2024-11-24 02:46:44,355 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-24 02:46:44,416 main DEBUG Took 0.055881 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 02:46:44,417 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 02:46:44,418 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 02:46:44,420 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 02:46:44,429 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,470 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 02:46:44,517 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,523 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,533 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,534 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,540 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,541 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,545 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,548 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,549 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,561 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,562 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,564 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,569 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,572 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,574 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,575 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,579 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,582 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 02:46:44,588 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,596 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 02:46:44,600 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 02:46:44,602 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 02:46:44,605 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 02:46:44,605 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 02:46:44,607 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 02:46:44,608 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 02:46:44,628 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 02:46:44,634 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 02:46:44,637 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 02:46:44,638 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 02:46:44,639 main DEBUG createAppenders(={Console}) 2024-11-24 02:46:44,640 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b initialized 2024-11-24 02:46:44,642 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b 2024-11-24 02:46:44,642 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@4b8d604b OK. 2024-11-24 02:46:44,643 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 02:46:44,644 main DEBUG OutputStream closed 2024-11-24 02:46:44,646 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 02:46:44,646 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 02:46:44,648 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@37d4349f OK 2024-11-24 02:46:44,811 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 02:46:44,814 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 02:46:44,816 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 02:46:44,817 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 02:46:44,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 02:46:44,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 02:46:44,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 02:46:44,819 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 02:46:44,820 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 02:46:44,820 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 02:46:44,820 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 02:46:44,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 02:46:44,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 02:46:44,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 02:46:44,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 02:46:44,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 02:46:44,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 02:46:44,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 02:46:44,827 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 02:46:44,827 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4331d187) with optional ClassLoader: null 2024-11-24 02:46:44,828 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 02:46:44,836 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4331d187] started OK. 2024-11-24T02:46:44,859 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync timeout: 13 mins 2024-11-24 02:46:44,862 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 02:46:44,862 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T02:46:44,972 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=11, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=888, ProcessCount=11, AvailableMemoryMB=11238 2024-11-24T02:46:45,393 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea 2024-11-24T02:46:46,370 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T02:46:46,602 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T02:46:46,787 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=wal, suffix=, logDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync, archiveDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/oldWALs, maxLogs=55 2024-11-24T02:46:46,877 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416406853 2024-11-24T02:46:47,028 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416406853 2024-11-24T02:46:47,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:47,218 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-24T02:46:47,220 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-24T02:46:47,286 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e6e36326512c58b59a3d26a05c314372, NAME => 'testLockupAroundBadAssignSync,,1732416407245.e6e36326512c58b59a3d26a05c314372.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testLockupAroundBadAssignSync', {TABLE_ATTRIBUTES => {DURABILITY => 'SYNC_WAL', READONLY => 'false'}}, {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}}, regionDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea 2024-11-24T02:46:47,420 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T02:46:47,443 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testLockupAroundBadAssignSync,,1732416407245.e6e36326512c58b59a3d26a05c314372.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T02:46:47,612 INFO [StoreOpener-e6e36326512c58b59a3d26a05c314372-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family MyCF of region e6e36326512c58b59a3d26a05c314372 2024-11-24T02:46:47,673 INFO [StoreOpener-e6e36326512c58b59a3d26a05c314372-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6e36326512c58b59a3d26a05c314372 columnFamilyName MyCF 2024-11-24T02:46:47,690 DEBUG [StoreOpener-e6e36326512c58b59a3d26a05c314372-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T02:46:47,700 INFO [StoreOpener-e6e36326512c58b59a3d26a05c314372-1 {}] regionserver.HStore(327): Store=e6e36326512c58b59a3d26a05c314372/MyCF, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T02:46:47,704 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e6e36326512c58b59a3d26a05c314372 2024-11-24T02:46:47,712 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/data/default/testLockupAroundBadAssignSync/e6e36326512c58b59a3d26a05c314372 2024-11-24T02:46:47,716 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/data/default/testLockupAroundBadAssignSync/e6e36326512c58b59a3d26a05c314372 2024-11-24T02:46:47,720 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e6e36326512c58b59a3d26a05c314372 2024-11-24T02:46:47,721 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e6e36326512c58b59a3d26a05c314372 2024-11-24T02:46:47,733 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e6e36326512c58b59a3d26a05c314372 2024-11-24T02:46:47,822 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/data/default/testLockupAroundBadAssignSync/e6e36326512c58b59a3d26a05c314372/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T02:46:47,823 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e6e36326512c58b59a3d26a05c314372; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64531458, jitterRate=-0.03840634226799011}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T02:46:47,838 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T02:46:47,894 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e6e36326512c58b59a3d26a05c314372: Writing region info on filesystem at 1732416407545Initializing all the Stores at 1732416407560 (+15 ms)Instantiating store for column family {NAME => 'MyCF', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)', METADATA => {'IN_MEMORY_COMPACTION' => 'NONE'}} at 1732416407561 (+1 ms)Cleaning up temporary data from old regions at 1732416407721 (+160 ms)Region opened successfully at 1732416407886 (+165 ms) 2024-11-24T02:46:47,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:47,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:47,965 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416406853) roll requested 2024-11-24T02:46:47,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416407969 2024-11-24T02:46:47,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:47,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:47,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:47,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:47,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:47,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416406853 with entries=1, filesize=249 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416407969 2024-11-24T02:46:48,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416407969) roll requested 2024-11-24T02:46:48,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408013 2024-11-24T02:46:48,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416407969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408013 2024-11-24T02:46:48,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408013) roll requested 2024-11-24T02:46:48,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408066 2024-11-24T02:46:48,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408066 2024-11-24T02:46:48,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408066) roll requested 2024-11-24T02:46:48,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408094 2024-11-24T02:46:48,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408094 2024-11-24T02:46:48,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408094) roll requested 2024-11-24T02:46:48,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408113 2024-11-24T02:46:48,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408113 2024-11-24T02:46:48,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408113) roll requested 2024-11-24T02:46:48,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408139 2024-11-24T02:46:48,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408139 2024-11-24T02:46:48,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,171 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408139) roll requested 2024-11-24T02:46:48,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408172 2024-11-24T02:46:48,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408139 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408172 2024-11-24T02:46:48,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408172) roll requested 2024-11-24T02:46:48,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408193 2024-11-24T02:46:48,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408193 2024-11-24T02:46:48,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408193) roll requested 2024-11-24T02:46:48,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408222 2024-11-24T02:46:48,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408222 2024-11-24T02:46:48,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408222) roll requested 2024-11-24T02:46:48,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408254 2024-11-24T02:46:48,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408254 2024-11-24T02:46:48,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408254) roll requested 2024-11-24T02:46:48,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408279 2024-11-24T02:46:48,309 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,309 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408279 2024-11-24T02:46:48,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408279) roll requested 2024-11-24T02:46:48,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408313 2024-11-24T02:46:48,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408313 2024-11-24T02:46:48,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408313) roll requested 2024-11-24T02:46:48,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408389 2024-11-24T02:46:48,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408389 2024-11-24T02:46:48,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408389) roll requested 2024-11-24T02:46:48,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408478 2024-11-24T02:46:48,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408478 2024-11-24T02:46:48,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408478) roll requested 2024-11-24T02:46:48,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408554 2024-11-24T02:46:48,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408554 2024-11-24T02:46:48,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408554) roll requested 2024-11-24T02:46:48,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408616 2024-11-24T02:46:48,637 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408616 2024-11-24T02:46:48,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408616) roll requested 2024-11-24T02:46:48,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408665 2024-11-24T02:46:48,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408665 2024-11-24T02:46:48,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408665) roll requested 2024-11-24T02:46:48,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408710 2024-11-24T02:46:48,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408710 2024-11-24T02:46:48,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408710) roll requested 2024-11-24T02:46:48,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408726 2024-11-24T02:46:48,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408726 2024-11-24T02:46:48,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408726) roll requested 2024-11-24T02:46:48,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408793 2024-11-24T02:46:48,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408793 2024-11-24T02:46:48,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408793) roll requested 2024-11-24T02:46:48,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408823 2024-11-24T02:46:48,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,831 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,831 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408823 2024-11-24T02:46:48,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408823) roll requested 2024-11-24T02:46:48,846 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408845 2024-11-24T02:46:48,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408845 2024-11-24T02:46:48,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,877 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,877 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408845) roll requested 2024-11-24T02:46:48,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408878 2024-11-24T02:46:48,893 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408878 2024-11-24T02:46:48,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,911 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408878) roll requested 2024-11-24T02:46:48,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408912 2024-11-24T02:46:48,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,932 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408912 2024-11-24T02:46:48,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408912) roll requested 2024-11-24T02:46:48,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408945 2024-11-24T02:46:48,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:48,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408945 2024-11-24T02:46:48,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:48,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:48,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408945) roll requested 2024-11-24T02:46:48,996 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416408996 2024-11-24T02:46:49,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408996 2024-11-24T02:46:49,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416408996) roll requested 2024-11-24T02:46:49,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409018 2024-11-24T02:46:49,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416408996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409018 2024-11-24T02:46:49,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409018) roll requested 2024-11-24T02:46:49,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409055 2024-11-24T02:46:49,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409055 2024-11-24T02:46:49,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409055) roll requested 2024-11-24T02:46:49,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409106 2024-11-24T02:46:49,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409106 2024-11-24T02:46:49,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409106) roll requested 2024-11-24T02:46:49,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409145 2024-11-24T02:46:49,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409145 2024-11-24T02:46:49,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409145) roll requested 2024-11-24T02:46:49,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409185 2024-11-24T02:46:49,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409185 2024-11-24T02:46:49,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409185) roll requested 2024-11-24T02:46:49,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409231 2024-11-24T02:46:49,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,267 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409231 2024-11-24T02:46:49,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409231) roll requested 2024-11-24T02:46:49,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409307 2024-11-24T02:46:49,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409307 2024-11-24T02:46:49,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,385 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,385 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409307) roll requested 2024-11-24T02:46:49,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409386 2024-11-24T02:46:49,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409307 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409386 2024-11-24T02:46:49,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,433 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409386) roll requested 2024-11-24T02:46:49,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409434 2024-11-24T02:46:49,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409434 2024-11-24T02:46:49,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409434) roll requested 2024-11-24T02:46:49,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409465 2024-11-24T02:46:49,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409465 2024-11-24T02:46:49,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409465) roll requested 2024-11-24T02:46:49,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409506 2024-11-24T02:46:49,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409506 2024-11-24T02:46:49,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409506) roll requested 2024-11-24T02:46:49,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409553 2024-11-24T02:46:49,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,570 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409506 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409553 2024-11-24T02:46:49,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,577 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409553) roll requested 2024-11-24T02:46:49,578 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409578 2024-11-24T02:46:49,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409578 2024-11-24T02:46:49,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409578) roll requested 2024-11-24T02:46:49,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409633 2024-11-24T02:46:49,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409578 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409633 2024-11-24T02:46:49,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409633) roll requested 2024-11-24T02:46:49,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409674 2024-11-24T02:46:49,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409674 2024-11-24T02:46:49,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,697 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409674) roll requested 2024-11-24T02:46:49,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409698 2024-11-24T02:46:49,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409698 2024-11-24T02:46:49,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409698) roll requested 2024-11-24T02:46:49,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409753 2024-11-24T02:46:49,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409753 2024-11-24T02:46:49,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,780 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409753) roll requested 2024-11-24T02:46:49,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409781 2024-11-24T02:46:49,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409781 2024-11-24T02:46:49,814 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409781) roll requested 2024-11-24T02:46:49,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409815 2024-11-24T02:46:49,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409815 2024-11-24T02:46:49,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,850 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409815) roll requested 2024-11-24T02:46:49,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409850 2024-11-24T02:46:49,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409815 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409850 2024-11-24T02:46:49,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409850) roll requested 2024-11-24T02:46:49,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409870 2024-11-24T02:46:49,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409850 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409870 2024-11-24T02:46:49,879 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,879 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409870) roll requested 2024-11-24T02:46:49,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409880 2024-11-24T02:46:49,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,887 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,887 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409880 2024-11-24T02:46:49,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409880) roll requested 2024-11-24T02:46:49,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409890 2024-11-24T02:46:49,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409880 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409890 2024-11-24T02:46:49,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409890) roll requested 2024-11-24T02:46:49,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409913 2024-11-24T02:46:49,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409913 2024-11-24T02:46:49,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409913) roll requested 2024-11-24T02:46:49,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409921 2024-11-24T02:46:49,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,932 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409921 2024-11-24T02:46:49,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409921) roll requested 2024-11-24T02:46:49,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409950 2024-11-24T02:46:49,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:49,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409950 2024-11-24T02:46:49,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:49,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:49,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409950) roll requested 2024-11-24T02:46:49,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416409989 2024-11-24T02:46:50,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409989 2024-11-24T02:46:50,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416409989) roll requested 2024-11-24T02:46:50,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410013 2024-11-24T02:46:50,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416409989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410013 2024-11-24T02:46:50,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410013) roll requested 2024-11-24T02:46:50,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410046 2024-11-24T02:46:50,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410046 2024-11-24T02:46:50,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410046) roll requested 2024-11-24T02:46:50,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410072 2024-11-24T02:46:50,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410072 2024-11-24T02:46:50,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410072) roll requested 2024-11-24T02:46:50,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410096 2024-11-24T02:46:50,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410096 2024-11-24T02:46:50,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410096) roll requested 2024-11-24T02:46:50,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410113 2024-11-24T02:46:50,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410113 2024-11-24T02:46:50,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410113) roll requested 2024-11-24T02:46:50,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410149 2024-11-24T02:46:50,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410149 2024-11-24T02:46:50,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,177 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410149) roll requested 2024-11-24T02:46:50,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410177 2024-11-24T02:46:50,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410177 2024-11-24T02:46:50,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410177) roll requested 2024-11-24T02:46:50,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410197 2024-11-24T02:46:50,207 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410197 2024-11-24T02:46:50,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410197) roll requested 2024-11-24T02:46:50,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410217 2024-11-24T02:46:50,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410217 2024-11-24T02:46:50,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410217) roll requested 2024-11-24T02:46:50,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410261 2024-11-24T02:46:50,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410217 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410261 2024-11-24T02:46:50,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410261) roll requested 2024-11-24T02:46:50,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410318 2024-11-24T02:46:50,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410318 2024-11-24T02:46:50,337 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,337 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,338 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410318) roll requested 2024-11-24T02:46:50,338 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410338 2024-11-24T02:46:50,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410338 2024-11-24T02:46:50,357 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,357 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,357 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410338) roll requested 2024-11-24T02:46:50,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410358 2024-11-24T02:46:50,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,373 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,373 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,373 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410338 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410358 2024-11-24T02:46:50,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410358) roll requested 2024-11-24T02:46:50,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410393 2024-11-24T02:46:50,409 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410393 2024-11-24T02:46:50,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410393) roll requested 2024-11-24T02:46:50,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410417 2024-11-24T02:46:50,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410417 2024-11-24T02:46:50,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410417) roll requested 2024-11-24T02:46:50,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410438 2024-11-24T02:46:50,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410438 2024-11-24T02:46:50,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410438) roll requested 2024-11-24T02:46:50,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410456 2024-11-24T02:46:50,462 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410438 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410456 2024-11-24T02:46:50,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410456) roll requested 2024-11-24T02:46:50,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410464 2024-11-24T02:46:50,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410464 2024-11-24T02:46:50,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410464) roll requested 2024-11-24T02:46:50,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410473 2024-11-24T02:46:50,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410473 2024-11-24T02:46:50,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410473) roll requested 2024-11-24T02:46:50,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410485 2024-11-24T02:46:50,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410485 2024-11-24T02:46:50,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410485) roll requested 2024-11-24T02:46:50,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410492 2024-11-24T02:46:50,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410492 2024-11-24T02:46:50,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410492) roll requested 2024-11-24T02:46:50,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410499 2024-11-24T02:46:50,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410499 2024-11-24T02:46:50,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,511 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410499) roll requested 2024-11-24T02:46:50,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410511 2024-11-24T02:46:50,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410511 2024-11-24T02:46:50,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,519 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410511) roll requested 2024-11-24T02:46:50,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410520 2024-11-24T02:46:50,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410520 2024-11-24T02:46:50,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,529 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,529 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410520) roll requested 2024-11-24T02:46:50,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410530 2024-11-24T02:46:50,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410530 2024-11-24T02:46:50,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,536 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410530) roll requested 2024-11-24T02:46:50,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410536 2024-11-24T02:46:50,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410536 2024-11-24T02:46:50,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410536) roll requested 2024-11-24T02:46:50,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410545 2024-11-24T02:46:50,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410545 2024-11-24T02:46:50,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410545) roll requested 2024-11-24T02:46:50,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410560 2024-11-24T02:46:50,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,570 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410560 2024-11-24T02:46:50,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410560) roll requested 2024-11-24T02:46:50,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410572 2024-11-24T02:46:50,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410572 2024-11-24T02:46:50,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410572) roll requested 2024-11-24T02:46:50,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410607 2024-11-24T02:46:50,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,630 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,630 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410607 2024-11-24T02:46:50,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410607) roll requested 2024-11-24T02:46:50,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410632 2024-11-24T02:46:50,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410632 2024-11-24T02:46:50,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410632) roll requested 2024-11-24T02:46:50,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410650 2024-11-24T02:46:50,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410650 2024-11-24T02:46:50,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410650) roll requested 2024-11-24T02:46:50,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410667 2024-11-24T02:46:50,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410650 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410667 2024-11-24T02:46:50,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410667) roll requested 2024-11-24T02:46:50,707 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410707 2024-11-24T02:46:50,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410707 2024-11-24T02:46:50,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410707) roll requested 2024-11-24T02:46:50,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410714 2024-11-24T02:46:50,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410707 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410714 2024-11-24T02:46:50,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410714) roll requested 2024-11-24T02:46:50,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410726 2024-11-24T02:46:50,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410726 2024-11-24T02:46:50,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410726) roll requested 2024-11-24T02:46:50,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410744 2024-11-24T02:46:50,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410744 2024-11-24T02:46:50,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410744) roll requested 2024-11-24T02:46:50,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410753 2024-11-24T02:46:50,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410753 2024-11-24T02:46:50,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410753) roll requested 2024-11-24T02:46:50,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410765 2024-11-24T02:46:50,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410765 2024-11-24T02:46:50,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410765) roll requested 2024-11-24T02:46:50,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410773 2024-11-24T02:46:50,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410773 2024-11-24T02:46:50,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410773) roll requested 2024-11-24T02:46:50,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410792 2024-11-24T02:46:50,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410792 2024-11-24T02:46:50,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410792) roll requested 2024-11-24T02:46:50,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410816 2024-11-24T02:46:50,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410792 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410816 2024-11-24T02:46:50,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410816) roll requested 2024-11-24T02:46:50,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410825 2024-11-24T02:46:50,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410825 2024-11-24T02:46:50,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410825) roll requested 2024-11-24T02:46:50,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410837 2024-11-24T02:46:50,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410825 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410837 2024-11-24T02:46:50,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410837) roll requested 2024-11-24T02:46:50,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410848 2024-11-24T02:46:50,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410848 2024-11-24T02:46:50,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,856 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410848) roll requested 2024-11-24T02:46:50,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410857 2024-11-24T02:46:50,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410857 2024-11-24T02:46:50,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410857) roll requested 2024-11-24T02:46:50,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410869 2024-11-24T02:46:50,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410857 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410869 2024-11-24T02:46:50,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410869) roll requested 2024-11-24T02:46:50,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410882 2024-11-24T02:46:50,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410882 2024-11-24T02:46:50,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,891 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410882) roll requested 2024-11-24T02:46:50,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410891 2024-11-24T02:46:50,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410891 2024-11-24T02:46:50,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410891) roll requested 2024-11-24T02:46:50,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410900 2024-11-24T02:46:50,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,908 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,908 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410900 2024-11-24T02:46:50,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,909 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410900) roll requested 2024-11-24T02:46:50,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410909 2024-11-24T02:46:50,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410909 2024-11-24T02:46:50,918 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,918 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,918 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410909) roll requested 2024-11-24T02:46:50,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410918 2024-11-24T02:46:50,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410918 2024-11-24T02:46:50,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,929 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,929 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,929 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410918) roll requested 2024-11-24T02:46:50,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410929 2024-11-24T02:46:50,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410929 2024-11-24T02:46:50,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,946 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410929) roll requested 2024-11-24T02:46:50,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410947 2024-11-24T02:46:50,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410929 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410947 2024-11-24T02:46:50,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410947) roll requested 2024-11-24T02:46:50,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410961 2024-11-24T02:46:50,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410961 2024-11-24T02:46:50,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410961) roll requested 2024-11-24T02:46:50,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410973 2024-11-24T02:46:50,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410973 2024-11-24T02:46:50,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:50,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:50,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410973) roll requested 2024-11-24T02:46:50,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416410981 2024-11-24T02:46:50,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:50,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410973 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410981 2024-11-24T02:46:51,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416410981) roll requested 2024-11-24T02:46:51,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411001 2024-11-24T02:46:51,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416410981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411001 2024-11-24T02:46:51,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411001) roll requested 2024-11-24T02:46:51,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411009 2024-11-24T02:46:51,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,023 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411009 2024-11-24T02:46:51,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411009) roll requested 2024-11-24T02:46:51,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411032 2024-11-24T02:46:51,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411032 2024-11-24T02:46:51,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411032) roll requested 2024-11-24T02:46:51,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411046 2024-11-24T02:46:51,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411032 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411046 2024-11-24T02:46:51,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411046) roll requested 2024-11-24T02:46:51,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411058 2024-11-24T02:46:51,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411046 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411058 2024-11-24T02:46:51,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411058) roll requested 2024-11-24T02:46:51,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411074 2024-11-24T02:46:51,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411074 2024-11-24T02:46:51,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411074) roll requested 2024-11-24T02:46:51,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411085 2024-11-24T02:46:51,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411085 2024-11-24T02:46:51,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411085) roll requested 2024-11-24T02:46:51,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411095 2024-11-24T02:46:51,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411095 2024-11-24T02:46:51,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,109 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411095) roll requested 2024-11-24T02:46:51,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411110 2024-11-24T02:46:51,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411095 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411110 2024-11-24T02:46:51,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411110) roll requested 2024-11-24T02:46:51,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411120 2024-11-24T02:46:51,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411110 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411120 2024-11-24T02:46:51,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411120) roll requested 2024-11-24T02:46:51,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411134 2024-11-24T02:46:51,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411134 2024-11-24T02:46:51,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,145 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411134) roll requested 2024-11-24T02:46:51,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411146 2024-11-24T02:46:51,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411146 2024-11-24T02:46:51,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411146) roll requested 2024-11-24T02:46:51,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411153 2024-11-24T02:46:51,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411146 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411153 2024-11-24T02:46:51,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411153) roll requested 2024-11-24T02:46:51,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411165 2024-11-24T02:46:51,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411165 2024-11-24T02:46:51,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411165) roll requested 2024-11-24T02:46:51,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411173 2024-11-24T02:46:51,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411173 2024-11-24T02:46:51,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411173) roll requested 2024-11-24T02:46:51,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411181 2024-11-24T02:46:51,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411181 2024-11-24T02:46:51,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411181) roll requested 2024-11-24T02:46:51,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411190 2024-11-24T02:46:51,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411190 2024-11-24T02:46:51,198 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,198 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411190) roll requested 2024-11-24T02:46:51,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411198 2024-11-24T02:46:51,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411198 2024-11-24T02:46:51,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,212 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411198) roll requested 2024-11-24T02:46:51,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411213 2024-11-24T02:46:51,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411198 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411213 2024-11-24T02:46:51,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411213) roll requested 2024-11-24T02:46:51,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411223 2024-11-24T02:46:51,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411223 2024-11-24T02:46:51,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411223) roll requested 2024-11-24T02:46:51,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411239 2024-11-24T02:46:51,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411239 2024-11-24T02:46:51,255 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,255 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,255 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411239) roll requested 2024-11-24T02:46:51,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411256 2024-11-24T02:46:51,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411256 2024-11-24T02:46:51,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411256) roll requested 2024-11-24T02:46:51,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411270 2024-11-24T02:46:51,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411270 2024-11-24T02:46:51,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411270) roll requested 2024-11-24T02:46:51,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411279 2024-11-24T02:46:51,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411279 2024-11-24T02:46:51,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411279) roll requested 2024-11-24T02:46:51,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411293 2024-11-24T02:46:51,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411293 2024-11-24T02:46:51,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411293) roll requested 2024-11-24T02:46:51,302 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411301 2024-11-24T02:46:51,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411301 2024-11-24T02:46:51,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,310 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411301) roll requested 2024-11-24T02:46:51,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411311 2024-11-24T02:46:51,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411311 2024-11-24T02:46:51,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411311) roll requested 2024-11-24T02:46:51,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411319 2024-11-24T02:46:51,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411319 2024-11-24T02:46:51,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411319) roll requested 2024-11-24T02:46:51,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411327 2024-11-24T02:46:51,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411319 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411327 2024-11-24T02:46:51,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,337 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411327) roll requested 2024-11-24T02:46:51,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411337 2024-11-24T02:46:51,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411337 2024-11-24T02:46:51,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,344 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411337) roll requested 2024-11-24T02:46:51,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411344 2024-11-24T02:46:51,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411344 2024-11-24T02:46:51,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411344) roll requested 2024-11-24T02:46:51,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411373 2024-11-24T02:46:51,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411344 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411373 2024-11-24T02:46:51,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411373) roll requested 2024-11-24T02:46:51,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411389 2024-11-24T02:46:51,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411389 2024-11-24T02:46:51,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411389) roll requested 2024-11-24T02:46:51,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411401 2024-11-24T02:46:51,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411389 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411401 2024-11-24T02:46:51,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411401) roll requested 2024-11-24T02:46:51,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411421 2024-11-24T02:46:51,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411421 2024-11-24T02:46:51,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,437 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411421) roll requested 2024-11-24T02:46:51,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411437 2024-11-24T02:46:51,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411437 2024-11-24T02:46:51,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411437) roll requested 2024-11-24T02:46:51,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411449 2024-11-24T02:46:51,455 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,455 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411449 2024-11-24T02:46:51,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411449) roll requested 2024-11-24T02:46:51,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411461 2024-11-24T02:46:51,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411461 2024-11-24T02:46:51,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411461) roll requested 2024-11-24T02:46:51,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411472 2024-11-24T02:46:51,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411472 2024-11-24T02:46:51,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411472) roll requested 2024-11-24T02:46:51,482 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411481 2024-11-24T02:46:51,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,488 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,488 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,488 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411481 2024-11-24T02:46:51,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411481) roll requested 2024-11-24T02:46:51,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411493 2024-11-24T02:46:51,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411493 2024-11-24T02:46:51,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411493) roll requested 2024-11-24T02:46:51,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411505 2024-11-24T02:46:51,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411505 2024-11-24T02:46:51,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411505) roll requested 2024-11-24T02:46:51,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411513 2024-11-24T02:46:51,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411513 2024-11-24T02:46:51,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411513) roll requested 2024-11-24T02:46:51,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411525 2024-11-24T02:46:51,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411525 2024-11-24T02:46:51,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411525) roll requested 2024-11-24T02:46:51,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411541 2024-11-24T02:46:51,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411541 2024-11-24T02:46:51,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411541) roll requested 2024-11-24T02:46:51,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411553 2024-11-24T02:46:51,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411553 2024-11-24T02:46:51,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,565 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411553) roll requested 2024-11-24T02:46:51,565 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411565 2024-11-24T02:46:51,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411565 2024-11-24T02:46:51,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411565) roll requested 2024-11-24T02:46:51,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411581 2024-11-24T02:46:51,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411565 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411581 2024-11-24T02:46:51,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411581) roll requested 2024-11-24T02:46:51,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411604 2024-11-24T02:46:51,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411604 2024-11-24T02:46:51,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,617 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411604) roll requested 2024-11-24T02:46:51,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411617 2024-11-24T02:46:51,627 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411604 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411617 2024-11-24T02:46:51,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411617) roll requested 2024-11-24T02:46:51,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411632 2024-11-24T02:46:51,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411632 2024-11-24T02:46:51,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411632) roll requested 2024-11-24T02:46:51,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411655 2024-11-24T02:46:51,667 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411655 2024-11-24T02:46:51,673 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,673 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,673 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411655) roll requested 2024-11-24T02:46:51,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411674 2024-11-24T02:46:51,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411674 2024-11-24T02:46:51,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411674) roll requested 2024-11-24T02:46:51,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411697 2024-11-24T02:46:51,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411697 2024-11-24T02:46:51,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411697) roll requested 2024-11-24T02:46:51,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411717 2024-11-24T02:46:51,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411717 2024-11-24T02:46:51,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411717) roll requested 2024-11-24T02:46:51,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411737 2024-11-24T02:46:51,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411737 2024-11-24T02:46:51,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411737) roll requested 2024-11-24T02:46:51,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411744 2024-11-24T02:46:51,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411744 2024-11-24T02:46:51,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411744) roll requested 2024-11-24T02:46:51,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411760 2024-11-24T02:46:51,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411760 2024-11-24T02:46:51,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411760) roll requested 2024-11-24T02:46:51,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411773 2024-11-24T02:46:51,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,786 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,786 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,786 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411773 2024-11-24T02:46:51,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411773) roll requested 2024-11-24T02:46:51,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411797 2024-11-24T02:46:51,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,814 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,814 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,814 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411797 2024-11-24T02:46:51,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411797) roll requested 2024-11-24T02:46:51,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411821 2024-11-24T02:46:51,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411821 2024-11-24T02:46:51,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411821) roll requested 2024-11-24T02:46:51,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411837 2024-11-24T02:46:51,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411837 2024-11-24T02:46:51,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,887 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411837) roll requested 2024-11-24T02:46:51,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411888 2024-11-24T02:46:51,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411888 2024-11-24T02:46:51,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411888) roll requested 2024-11-24T02:46:51,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411921 2024-11-24T02:46:51,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411888 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411921 2024-11-24T02:46:51,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:51,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:51,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411921) roll requested 2024-11-24T02:46:51,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416411958 2024-11-24T02:46:51,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:51,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411958 2024-11-24T02:46:52,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,005 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416411958) roll requested 2024-11-24T02:46:52,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412005 2024-11-24T02:46:52,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416411958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412005 2024-11-24T02:46:52,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,040 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412005) roll requested 2024-11-24T02:46:52,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412041 2024-11-24T02:46:52,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412005 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412041 2024-11-24T02:46:52,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412041) roll requested 2024-11-24T02:46:52,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412073 2024-11-24T02:46:52,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412073 2024-11-24T02:46:52,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412073) roll requested 2024-11-24T02:46:52,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412093 2024-11-24T02:46:52,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412093 2024-11-24T02:46:52,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412093) roll requested 2024-11-24T02:46:52,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412106 2024-11-24T02:46:52,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412106 2024-11-24T02:46:52,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,125 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412106) roll requested 2024-11-24T02:46:52,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412125 2024-11-24T02:46:52,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412125 2024-11-24T02:46:52,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412125) roll requested 2024-11-24T02:46:52,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412137 2024-11-24T02:46:52,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412125 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412137 2024-11-24T02:46:52,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412137) roll requested 2024-11-24T02:46:52,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412148 2024-11-24T02:46:52,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412148 2024-11-24T02:46:52,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412148) roll requested 2024-11-24T02:46:52,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412165 2024-11-24T02:46:52,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412165 2024-11-24T02:46:52,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412165) roll requested 2024-11-24T02:46:52,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412177 2024-11-24T02:46:52,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412177 2024-11-24T02:46:52,186 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,186 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412177) roll requested 2024-11-24T02:46:52,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412186 2024-11-24T02:46:52,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412186 2024-11-24T02:46:52,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412186) roll requested 2024-11-24T02:46:52,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412197 2024-11-24T02:46:52,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412197 2024-11-24T02:46:52,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,209 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412197) roll requested 2024-11-24T02:46:52,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412209 2024-11-24T02:46:52,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412209 2024-11-24T02:46:52,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412209) roll requested 2024-11-24T02:46:52,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412225 2024-11-24T02:46:52,232 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,232 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,232 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,232 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,232 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412209 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412225 2024-11-24T02:46:52,233 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,233 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,233 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412225) roll requested 2024-11-24T02:46:52,234 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412234 2024-11-24T02:46:52,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412234 2024-11-24T02:46:52,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412234) roll requested 2024-11-24T02:46:52,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412245 2024-11-24T02:46:52,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412234 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412245 2024-11-24T02:46:52,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412245) roll requested 2024-11-24T02:46:52,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412260 2024-11-24T02:46:52,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412260 2024-11-24T02:46:52,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412260) roll requested 2024-11-24T02:46:52,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412269 2024-11-24T02:46:52,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412269 2024-11-24T02:46:52,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412269) roll requested 2024-11-24T02:46:52,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412277 2024-11-24T02:46:52,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412277 2024-11-24T02:46:52,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412277) roll requested 2024-11-24T02:46:52,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412284 2024-11-24T02:46:52,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412284 2024-11-24T02:46:52,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,298 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412284) roll requested 2024-11-24T02:46:52,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412298 2024-11-24T02:46:52,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412298 2024-11-24T02:46:52,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,368 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412298) roll requested 2024-11-24T02:46:52,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412368 2024-11-24T02:46:52,375 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,376 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412368 2024-11-24T02:46:52,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412368) roll requested 2024-11-24T02:46:52,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412378 2024-11-24T02:46:52,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412368 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412378 2024-11-24T02:46:52,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412378) roll requested 2024-11-24T02:46:52,395 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412394 2024-11-24T02:46:52,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412394 2024-11-24T02:46:52,414 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,414 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,414 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,414 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412394) roll requested 2024-11-24T02:46:52,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412414 2024-11-24T02:46:52,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412414 2024-11-24T02:46:52,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,423 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412414) roll requested 2024-11-24T02:46:52,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412424 2024-11-24T02:46:52,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412414 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412424 2024-11-24T02:46:52,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412424) roll requested 2024-11-24T02:46:52,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412434 2024-11-24T02:46:52,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412434 2024-11-24T02:46:52,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412434) roll requested 2024-11-24T02:46:52,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412445 2024-11-24T02:46:52,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412445 2024-11-24T02:46:52,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412445) roll requested 2024-11-24T02:46:52,455 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412453 2024-11-24T02:46:52,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412453 2024-11-24T02:46:52,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412453) roll requested 2024-11-24T02:46:52,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412468 2024-11-24T02:46:52,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412468 2024-11-24T02:46:52,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412468) roll requested 2024-11-24T02:46:52,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412480 2024-11-24T02:46:52,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412480 2024-11-24T02:46:52,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412480) roll requested 2024-11-24T02:46:52,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412489 2024-11-24T02:46:52,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412489 2024-11-24T02:46:52,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412489) roll requested 2024-11-24T02:46:52,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412496 2024-11-24T02:46:52,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412489 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412496 2024-11-24T02:46:52,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412496) roll requested 2024-11-24T02:46:52,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412503 2024-11-24T02:46:52,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412503 2024-11-24T02:46:52,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412503) roll requested 2024-11-24T02:46:52,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412517 2024-11-24T02:46:52,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412503 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412517 2024-11-24T02:46:52,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412517) roll requested 2024-11-24T02:46:52,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412526 2024-11-24T02:46:52,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412526 2024-11-24T02:46:52,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412526) roll requested 2024-11-24T02:46:52,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412547 2024-11-24T02:46:52,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412547 2024-11-24T02:46:52,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412547) roll requested 2024-11-24T02:46:52,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412556 2024-11-24T02:46:52,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412556 2024-11-24T02:46:52,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412556) roll requested 2024-11-24T02:46:52,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412569 2024-11-24T02:46:52,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412569 2024-11-24T02:46:52,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412569) roll requested 2024-11-24T02:46:52,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412576 2024-11-24T02:46:52,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412576 2024-11-24T02:46:52,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412576) roll requested 2024-11-24T02:46:52,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412583 2024-11-24T02:46:52,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412583 2024-11-24T02:46:52,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412583) roll requested 2024-11-24T02:46:52,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412593 2024-11-24T02:46:52,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412593 2024-11-24T02:46:52,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412593) roll requested 2024-11-24T02:46:52,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412603 2024-11-24T02:46:52,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412603 2024-11-24T02:46:52,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412603) roll requested 2024-11-24T02:46:52,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412614 2024-11-24T02:46:52,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412614 2024-11-24T02:46:52,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412614) roll requested 2024-11-24T02:46:52,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412624 2024-11-24T02:46:52,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412624 2024-11-24T02:46:52,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412624) roll requested 2024-11-24T02:46:52,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412638 2024-11-24T02:46:52,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412638 2024-11-24T02:46:52,651 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,651 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412638) roll requested 2024-11-24T02:46:52,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412651 2024-11-24T02:46:52,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,658 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,658 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,658 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,658 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,658 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412651 2024-11-24T02:46:52,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412651) roll requested 2024-11-24T02:46:52,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412660 2024-11-24T02:46:52,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412660 2024-11-24T02:46:52,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412660) roll requested 2024-11-24T02:46:52,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412668 2024-11-24T02:46:52,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412668 2024-11-24T02:46:52,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412668) roll requested 2024-11-24T02:46:52,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412677 2024-11-24T02:46:52,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,687 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,687 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,687 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,687 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,687 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412677 2024-11-24T02:46:52,688 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,688 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,688 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412677) roll requested 2024-11-24T02:46:52,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412688 2024-11-24T02:46:52,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412688 2024-11-24T02:46:52,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,695 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,695 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412688) roll requested 2024-11-24T02:46:52,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412695 2024-11-24T02:46:52,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412688 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412695 2024-11-24T02:46:52,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,702 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412695) roll requested 2024-11-24T02:46:52,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412703 2024-11-24T02:46:52,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412703 2024-11-24T02:46:52,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412703) roll requested 2024-11-24T02:46:52,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412725 2024-11-24T02:46:52,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412725 2024-11-24T02:46:52,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,749 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412725) roll requested 2024-11-24T02:46:52,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412750 2024-11-24T02:46:52,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412750 2024-11-24T02:46:52,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412750) roll requested 2024-11-24T02:46:52,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412805 2024-11-24T02:46:52,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412805 2024-11-24T02:46:52,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412805) roll requested 2024-11-24T02:46:52,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412841 2024-11-24T02:46:52,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412841 2024-11-24T02:46:52,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412841) roll requested 2024-11-24T02:46:52,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412879 2024-11-24T02:46:52,892 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,892 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,893 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,893 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,893 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,893 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412841 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412879 2024-11-24T02:46:52,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412879) roll requested 2024-11-24T02:46:52,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412897 2024-11-24T02:46:52,908 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,909 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,909 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412897 2024-11-24T02:46:52,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412897) roll requested 2024-11-24T02:46:52,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412913 2024-11-24T02:46:52,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412913 2024-11-24T02:46:52,932 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,932 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412913) roll requested 2024-11-24T02:46:52,933 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412933 2024-11-24T02:46:52,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412913 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412933 2024-11-24T02:46:52,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412933) roll requested 2024-11-24T02:46:52,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412945 2024-11-24T02:46:52,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,955 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,955 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412933 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412945 2024-11-24T02:46:52,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412945) roll requested 2024-11-24T02:46:52,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412959 2024-11-24T02:46:52,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412959 2024-11-24T02:46:52,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412959) roll requested 2024-11-24T02:46:52,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412968 2024-11-24T02:46:52,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412968 2024-11-24T02:46:52,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412968) roll requested 2024-11-24T02:46:52,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412982 2024-11-24T02:46:52,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412982 2024-11-24T02:46:52,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412982) roll requested 2024-11-24T02:46:52,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412992 2024-11-24T02:46:52,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:52,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412992 2024-11-24T02:46:52,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:52,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:52,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412992) roll requested 2024-11-24T02:46:52,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416412998 2024-11-24T02:46:53,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412998 2024-11-24T02:46:53,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416412998) roll requested 2024-11-24T02:46:53,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413009 2024-11-24T02:46:53,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416412998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413009 2024-11-24T02:46:53,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413009) roll requested 2024-11-24T02:46:53,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413029 2024-11-24T02:46:53,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413029 2024-11-24T02:46:53,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,037 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413029) roll requested 2024-11-24T02:46:53,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413037 2024-11-24T02:46:53,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413037 2024-11-24T02:46:53,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413037) roll requested 2024-11-24T02:46:53,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413051 2024-11-24T02:46:53,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413051 2024-11-24T02:46:53,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413051) roll requested 2024-11-24T02:46:53,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413061 2024-11-24T02:46:53,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413061 2024-11-24T02:46:53,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,071 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413061) roll requested 2024-11-24T02:46:53,071 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413071 2024-11-24T02:46:53,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413071 2024-11-24T02:46:53,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,079 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413071) roll requested 2024-11-24T02:46:53,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413079 2024-11-24T02:46:53,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413071 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413079 2024-11-24T02:46:53,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,111 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,111 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413079) roll requested 2024-11-24T02:46:53,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413111 2024-11-24T02:46:53,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413079 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413111 2024-11-24T02:46:53,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413111) roll requested 2024-11-24T02:46:53,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413145 2024-11-24T02:46:53,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413111 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413145 2024-11-24T02:46:53,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413145) roll requested 2024-11-24T02:46:53,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413167 2024-11-24T02:46:53,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413167 2024-11-24T02:46:53,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413167) roll requested 2024-11-24T02:46:53,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413197 2024-11-24T02:46:53,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413197 2024-11-24T02:46:53,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413197) roll requested 2024-11-24T02:46:53,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413210 2024-11-24T02:46:53,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413210 2024-11-24T02:46:53,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413210) roll requested 2024-11-24T02:46:53,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413223 2024-11-24T02:46:53,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413223 2024-11-24T02:46:53,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413223) roll requested 2024-11-24T02:46:53,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413257 2024-11-24T02:46:53,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413223 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413257 2024-11-24T02:46:53,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413257) roll requested 2024-11-24T02:46:53,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413297 2024-11-24T02:46:53,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413297 2024-11-24T02:46:53,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413297) roll requested 2024-11-24T02:46:53,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413320 2024-11-24T02:46:53,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413320 2024-11-24T02:46:53,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413320) roll requested 2024-11-24T02:46:53,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413334 2024-11-24T02:46:53,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413334 2024-11-24T02:46:53,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413334) roll requested 2024-11-24T02:46:53,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413349 2024-11-24T02:46:53,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413349 2024-11-24T02:46:53,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413349) roll requested 2024-11-24T02:46:53,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413356 2024-11-24T02:46:53,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413356 2024-11-24T02:46:53,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,369 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413356) roll requested 2024-11-24T02:46:53,369 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413369 2024-11-24T02:46:53,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413369 2024-11-24T02:46:53,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,376 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,376 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413369) roll requested 2024-11-24T02:46:53,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413377 2024-11-24T02:46:53,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,382 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,382 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,382 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,382 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413369 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413377 2024-11-24T02:46:53,383 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,383 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,383 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,383 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413377) roll requested 2024-11-24T02:46:53,383 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413383 2024-11-24T02:46:53,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413383 2024-11-24T02:46:53,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,396 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413383) roll requested 2024-11-24T02:46:53,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413397 2024-11-24T02:46:53,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,410 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413383 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413397 2024-11-24T02:46:53,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413397) roll requested 2024-11-24T02:46:53,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413412 2024-11-24T02:46:53,418 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413412 2024-11-24T02:46:53,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413412) roll requested 2024-11-24T02:46:53,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413421 2024-11-24T02:46:53,428 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,428 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,428 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413412 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413421 2024-11-24T02:46:53,429 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,431 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413421) roll requested 2024-11-24T02:46:53,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413431 2024-11-24T02:46:53,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413431 2024-11-24T02:46:53,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413431) roll requested 2024-11-24T02:46:53,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413439 2024-11-24T02:46:53,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413439 2024-11-24T02:46:53,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413439) roll requested 2024-11-24T02:46:53,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413446 2024-11-24T02:46:53,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413446 2024-11-24T02:46:53,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,451 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,451 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413446) roll requested 2024-11-24T02:46:53,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413452 2024-11-24T02:46:53,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413446 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413452 2024-11-24T02:46:53,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413452) roll requested 2024-11-24T02:46:53,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413462 2024-11-24T02:46:53,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413452 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413462 2024-11-24T02:46:53,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413462) roll requested 2024-11-24T02:46:53,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413478 2024-11-24T02:46:53,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413478 2024-11-24T02:46:53,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413478) roll requested 2024-11-24T02:46:53,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413494 2024-11-24T02:46:53,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413494 2024-11-24T02:46:53,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,500 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,500 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413494) roll requested 2024-11-24T02:46:53,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413501 2024-11-24T02:46:53,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413501 2024-11-24T02:46:53,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413501) roll requested 2024-11-24T02:46:53,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413511 2024-11-24T02:46:53,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413511 2024-11-24T02:46:53,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413511) roll requested 2024-11-24T02:46:53,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413521 2024-11-24T02:46:53,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413521 2024-11-24T02:46:53,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413521) roll requested 2024-11-24T02:46:53,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413531 2024-11-24T02:46:53,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413531 2024-11-24T02:46:53,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413531) roll requested 2024-11-24T02:46:53,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413538 2024-11-24T02:46:53,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413538 2024-11-24T02:46:53,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413538) roll requested 2024-11-24T02:46:53,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413547 2024-11-24T02:46:53,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413547 2024-11-24T02:46:53,553 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,553 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413547) roll requested 2024-11-24T02:46:53,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413554 2024-11-24T02:46:53,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413554 2024-11-24T02:46:53,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413554) roll requested 2024-11-24T02:46:53,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413560 2024-11-24T02:46:53,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413560 2024-11-24T02:46:53,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413560) roll requested 2024-11-24T02:46:53,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413568 2024-11-24T02:46:53,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,576 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,576 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,576 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413568 2024-11-24T02:46:53,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413568) roll requested 2024-11-24T02:46:53,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413593 2024-11-24T02:46:53,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413593 2024-11-24T02:46:53,611 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,611 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413593) roll requested 2024-11-24T02:46:53,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413611 2024-11-24T02:46:53,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413611 2024-11-24T02:46:53,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413611) roll requested 2024-11-24T02:46:53,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413640 2024-11-24T02:46:53,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,664 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,664 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,664 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413640 2024-11-24T02:46:53,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413640) roll requested 2024-11-24T02:46:53,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413669 2024-11-24T02:46:53,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413669 2024-11-24T02:46:53,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413669) roll requested 2024-11-24T02:46:53,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413686 2024-11-24T02:46:53,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413686 2024-11-24T02:46:53,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413686) roll requested 2024-11-24T02:46:53,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413700 2024-11-24T02:46:53,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413700 2024-11-24T02:46:53,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,714 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413700) roll requested 2024-11-24T02:46:53,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413714 2024-11-24T02:46:53,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413714 2024-11-24T02:46:53,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413714) roll requested 2024-11-24T02:46:53,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413725 2024-11-24T02:46:53,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413725 2024-11-24T02:46:53,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413725) roll requested 2024-11-24T02:46:53,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413740 2024-11-24T02:46:53,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413740 2024-11-24T02:46:53,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413740) roll requested 2024-11-24T02:46:53,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413749 2024-11-24T02:46:53,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413749 2024-11-24T02:46:53,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413749) roll requested 2024-11-24T02:46:53,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413759 2024-11-24T02:46:53,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413759 2024-11-24T02:46:53,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413759) roll requested 2024-11-24T02:46:53,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413770 2024-11-24T02:46:53,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,776 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,776 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,776 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,776 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,776 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413770 2024-11-24T02:46:53,777 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,777 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,777 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413770) roll requested 2024-11-24T02:46:53,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413777 2024-11-24T02:46:53,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413777 2024-11-24T02:46:53,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413777) roll requested 2024-11-24T02:46:53,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413803 2024-11-24T02:46:53,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413803 2024-11-24T02:46:53,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413803) roll requested 2024-11-24T02:46:53,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413821 2024-11-24T02:46:53,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413821 2024-11-24T02:46:53,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,833 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413821) roll requested 2024-11-24T02:46:53,834 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413834 2024-11-24T02:46:53,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413834 2024-11-24T02:46:53,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413834) roll requested 2024-11-24T02:46:53,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413843 2024-11-24T02:46:53,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413834 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413843 2024-11-24T02:46:53,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413843) roll requested 2024-11-24T02:46:53,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413852 2024-11-24T02:46:53,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413852 2024-11-24T02:46:53,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413852) roll requested 2024-11-24T02:46:53,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413860 2024-11-24T02:46:53,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413860 2024-11-24T02:46:53,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,873 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413860) roll requested 2024-11-24T02:46:53,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413873 2024-11-24T02:46:53,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413873 2024-11-24T02:46:53,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,880 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413873) roll requested 2024-11-24T02:46:53,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413881 2024-11-24T02:46:53,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413881 2024-11-24T02:46:53,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413881) roll requested 2024-11-24T02:46:53,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413904 2024-11-24T02:46:53,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413904 2024-11-24T02:46:53,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,916 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413904) roll requested 2024-11-24T02:46:53,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413916 2024-11-24T02:46:53,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413916 2024-11-24T02:46:53,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413916) roll requested 2024-11-24T02:46:53,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413925 2024-11-24T02:46:53,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413916 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413925 2024-11-24T02:46:53,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,932 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,932 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413925) roll requested 2024-11-24T02:46:53,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413932 2024-11-24T02:46:53,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413925 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413932 2024-11-24T02:46:53,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413932) roll requested 2024-11-24T02:46:53,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413941 2024-11-24T02:46:53,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,947 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,947 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,947 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413932 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413941 2024-11-24T02:46:53,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413941) roll requested 2024-11-24T02:46:53,948 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413948 2024-11-24T02:46:53,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413941 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413948 2024-11-24T02:46:53,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413948) roll requested 2024-11-24T02:46:53,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413956 2024-11-24T02:46:53,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413956 2024-11-24T02:46:53,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413956) roll requested 2024-11-24T02:46:53,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413965 2024-11-24T02:46:53,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:53,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413956 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413965 2024-11-24T02:46:53,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:53,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:53,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413965) roll requested 2024-11-24T02:46:53,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416413986 2024-11-24T02:46:54,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413986 2024-11-24T02:46:54,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416413986) roll requested 2024-11-24T02:46:54,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414004 2024-11-24T02:46:54,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416413986 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414004 2024-11-24T02:46:54,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414004) roll requested 2024-11-24T02:46:54,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414014 2024-11-24T02:46:54,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414014 2024-11-24T02:46:54,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414014) roll requested 2024-11-24T02:46:54,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414029 2024-11-24T02:46:54,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414029 2024-11-24T02:46:54,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414029) roll requested 2024-11-24T02:46:54,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414041 2024-11-24T02:46:54,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,052 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,052 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414041 2024-11-24T02:46:54,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,052 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414041) roll requested 2024-11-24T02:46:54,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414053 2024-11-24T02:46:54,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414053 2024-11-24T02:46:54,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414053) roll requested 2024-11-24T02:46:54,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414061 2024-11-24T02:46:54,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414061 2024-11-24T02:46:54,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414061) roll requested 2024-11-24T02:46:54,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414069 2024-11-24T02:46:54,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414069 2024-11-24T02:46:54,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414069) roll requested 2024-11-24T02:46:54,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414080 2024-11-24T02:46:54,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414080 2024-11-24T02:46:54,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,089 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414080) roll requested 2024-11-24T02:46:54,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414089 2024-11-24T02:46:54,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414089 2024-11-24T02:46:54,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414089) roll requested 2024-11-24T02:46:54,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414097 2024-11-24T02:46:54,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414097 2024-11-24T02:46:54,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414097) roll requested 2024-11-24T02:46:54,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414105 2024-11-24T02:46:54,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414105 2024-11-24T02:46:54,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414105) roll requested 2024-11-24T02:46:54,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414112 2024-11-24T02:46:54,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414112 2024-11-24T02:46:54,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,119 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414112) roll requested 2024-11-24T02:46:54,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414119 2024-11-24T02:46:54,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414119 2024-11-24T02:46:54,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414119) roll requested 2024-11-24T02:46:54,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414128 2024-11-24T02:46:54,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414128 2024-11-24T02:46:54,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414128) roll requested 2024-11-24T02:46:54,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414135 2024-11-24T02:46:54,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414135 2024-11-24T02:46:54,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414135) roll requested 2024-11-24T02:46:54,144 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414144 2024-11-24T02:46:54,149 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,149 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,149 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,149 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,149 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414144 2024-11-24T02:46:54,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414144) roll requested 2024-11-24T02:46:54,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414152 2024-11-24T02:46:54,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414144 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414152 2024-11-24T02:46:54,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414152) roll requested 2024-11-24T02:46:54,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414159 2024-11-24T02:46:54,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414159 2024-11-24T02:46:54,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414159) roll requested 2024-11-24T02:46:54,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414167 2024-11-24T02:46:54,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414167 2024-11-24T02:46:54,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414167) roll requested 2024-11-24T02:46:54,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414180 2024-11-24T02:46:54,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414180 2024-11-24T02:46:54,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414180) roll requested 2024-11-24T02:46:54,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414189 2024-11-24T02:46:54,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414189 2024-11-24T02:46:54,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414189) roll requested 2024-11-24T02:46:54,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414197 2024-11-24T02:46:54,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414197 2024-11-24T02:46:54,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414197) roll requested 2024-11-24T02:46:54,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414208 2024-11-24T02:46:54,214 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,214 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,214 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,214 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,214 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,214 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414208 2024-11-24T02:46:54,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414208) roll requested 2024-11-24T02:46:54,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414216 2024-11-24T02:46:54,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414216 2024-11-24T02:46:54,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414216) roll requested 2024-11-24T02:46:54,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414224 2024-11-24T02:46:54,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414224 2024-11-24T02:46:54,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414224) roll requested 2024-11-24T02:46:54,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414232 2024-11-24T02:46:54,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414232 2024-11-24T02:46:54,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414232) roll requested 2024-11-24T02:46:54,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414239 2024-11-24T02:46:54,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414239 2024-11-24T02:46:54,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414239) roll requested 2024-11-24T02:46:54,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414246 2024-11-24T02:46:54,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414246 2024-11-24T02:46:54,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414246) roll requested 2024-11-24T02:46:54,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414257 2024-11-24T02:46:54,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414257 2024-11-24T02:46:54,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,263 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414257) roll requested 2024-11-24T02:46:54,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414263 2024-11-24T02:46:54,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414263 2024-11-24T02:46:54,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414263) roll requested 2024-11-24T02:46:54,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414272 2024-11-24T02:46:54,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414263 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414272 2024-11-24T02:46:54,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414272) roll requested 2024-11-24T02:46:54,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414285 2024-11-24T02:46:54,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414285 2024-11-24T02:46:54,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,293 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414285) roll requested 2024-11-24T02:46:54,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414293 2024-11-24T02:46:54,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414285 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414293 2024-11-24T02:46:54,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414293) roll requested 2024-11-24T02:46:54,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414312 2024-11-24T02:46:54,318 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,318 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414293 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414312 2024-11-24T02:46:54,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414312) roll requested 2024-11-24T02:46:54,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414320 2024-11-24T02:46:54,325 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,325 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,325 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,325 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,325 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414320 2024-11-24T02:46:54,326 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,326 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414320) roll requested 2024-11-24T02:46:54,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414327 2024-11-24T02:46:54,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414327 2024-11-24T02:46:54,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414327) roll requested 2024-11-24T02:46:54,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414336 2024-11-24T02:46:54,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414336 2024-11-24T02:46:54,351 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,351 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,351 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414336) roll requested 2024-11-24T02:46:54,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414351 2024-11-24T02:46:54,356 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,356 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,356 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,356 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,356 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414351 2024-11-24T02:46:54,358 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,358 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,358 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,358 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414351) roll requested 2024-11-24T02:46:54,358 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414358 2024-11-24T02:46:54,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414351 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414358 2024-11-24T02:46:54,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414358) roll requested 2024-11-24T02:46:54,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414367 2024-11-24T02:46:54,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,373 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414358 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414367 2024-11-24T02:46:54,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,373 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414367) roll requested 2024-11-24T02:46:54,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414373 2024-11-24T02:46:54,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414373 2024-11-24T02:46:54,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,379 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414373) roll requested 2024-11-24T02:46:54,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414380 2024-11-24T02:46:54,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414373 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414380 2024-11-24T02:46:54,388 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,388 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414380) roll requested 2024-11-24T02:46:54,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414388 2024-11-24T02:46:54,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414388 2024-11-24T02:46:54,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414388) roll requested 2024-11-24T02:46:54,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414407 2024-11-24T02:46:54,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414407 2024-11-24T02:46:54,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,420 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414407) roll requested 2024-11-24T02:46:54,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414421 2024-11-24T02:46:54,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414421 2024-11-24T02:46:54,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414421) roll requested 2024-11-24T02:46:54,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414436 2024-11-24T02:46:54,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414436 2024-11-24T02:46:54,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414436) roll requested 2024-11-24T02:46:54,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414457 2024-11-24T02:46:54,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414457 2024-11-24T02:46:54,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414457) roll requested 2024-11-24T02:46:54,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414491 2024-11-24T02:46:54,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414491 2024-11-24T02:46:54,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414491) roll requested 2024-11-24T02:46:54,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414511 2024-11-24T02:46:54,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414491 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414511 2024-11-24T02:46:54,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414511) roll requested 2024-11-24T02:46:54,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414569 2024-11-24T02:46:54,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414511 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414569 2024-11-24T02:46:54,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,593 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,593 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414569) roll requested 2024-11-24T02:46:54,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414593 2024-11-24T02:46:54,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414593 2024-11-24T02:46:54,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414593) roll requested 2024-11-24T02:46:54,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414603 2024-11-24T02:46:54,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414603 2024-11-24T02:46:54,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414603) roll requested 2024-11-24T02:46:54,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414625 2024-11-24T02:46:54,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414625 2024-11-24T02:46:54,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414625) roll requested 2024-11-24T02:46:54,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414637 2024-11-24T02:46:54,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414637 2024-11-24T02:46:54,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414637) roll requested 2024-11-24T02:46:54,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414645 2024-11-24T02:46:54,658 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414645 2024-11-24T02:46:54,665 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,665 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,665 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414645) roll requested 2024-11-24T02:46:54,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414665 2024-11-24T02:46:54,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414645 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414665 2024-11-24T02:46:54,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414665) roll requested 2024-11-24T02:46:54,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414676 2024-11-24T02:46:54,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414665 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414676 2024-11-24T02:46:54,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414676) roll requested 2024-11-24T02:46:54,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414684 2024-11-24T02:46:54,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414684 2024-11-24T02:46:54,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414684) roll requested 2024-11-24T02:46:54,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414702 2024-11-24T02:46:54,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414702 2024-11-24T02:46:54,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,723 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,723 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414702) roll requested 2024-11-24T02:46:54,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414724 2024-11-24T02:46:54,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414702 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414724 2024-11-24T02:46:54,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,734 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414724) roll requested 2024-11-24T02:46:54,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414735 2024-11-24T02:46:54,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,740 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,740 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,740 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414735 2024-11-24T02:46:54,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,742 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414735) roll requested 2024-11-24T02:46:54,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414742 2024-11-24T02:46:54,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414742 2024-11-24T02:46:54,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414742) roll requested 2024-11-24T02:46:54,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414752 2024-11-24T02:46:54,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414752 2024-11-24T02:46:54,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,770 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414752) roll requested 2024-11-24T02:46:54,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414770 2024-11-24T02:46:54,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414770 2024-11-24T02:46:54,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414770) roll requested 2024-11-24T02:46:54,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414779 2024-11-24T02:46:54,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414770 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414779 2024-11-24T02:46:54,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414779) roll requested 2024-11-24T02:46:54,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414797 2024-11-24T02:46:54,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414797 2024-11-24T02:46:54,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414797) roll requested 2024-11-24T02:46:54,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414822 2024-11-24T02:46:54,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414822 2024-11-24T02:46:54,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414822) roll requested 2024-11-24T02:46:54,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414831 2024-11-24T02:46:54,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,839 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414822 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414831 2024-11-24T02:46:54,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,840 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414831) roll requested 2024-11-24T02:46:54,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414840 2024-11-24T02:46:54,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414840 2024-11-24T02:46:54,852 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414831 is not closed yet, will try archiving it next time 2024-11-24T02:46:54,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414840) roll requested 2024-11-24T02:46:54,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414852 2024-11-24T02:46:54,883 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414852 2024-11-24T02:46:54,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414852) roll requested 2024-11-24T02:46:54,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414898 2024-11-24T02:46:54,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414898 2024-11-24T02:46:54,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414898) roll requested 2024-11-24T02:46:54,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414937 2024-11-24T02:46:54,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:54,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414898 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414937 2024-11-24T02:46:54,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:54,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:54,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414937) roll requested 2024-11-24T02:46:54,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416414989 2024-11-24T02:46:55,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,037 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,037 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414989 2024-11-24T02:46:55,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,053 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,053 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,053 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416414989) roll requested 2024-11-24T02:46:55,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415053 2024-11-24T02:46:55,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416414989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415053 2024-11-24T02:46:55,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415053) roll requested 2024-11-24T02:46:55,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415084 2024-11-24T02:46:55,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415053 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415084 2024-11-24T02:46:55,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415084) roll requested 2024-11-24T02:46:55,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415121 2024-11-24T02:46:55,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,130 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,130 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,130 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,130 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,130 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415121 2024-11-24T02:46:55,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415121) roll requested 2024-11-24T02:46:55,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415143 2024-11-24T02:46:55,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415143 2024-11-24T02:46:55,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415143) roll requested 2024-11-24T02:46:55,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415161 2024-11-24T02:46:55,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415161 2024-11-24T02:46:55,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415161) roll requested 2024-11-24T02:46:55,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415181 2024-11-24T02:46:55,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415181 2024-11-24T02:46:55,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415181) roll requested 2024-11-24T02:46:55,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415189 2024-11-24T02:46:55,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415189 2024-11-24T02:46:55,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415189) roll requested 2024-11-24T02:46:55,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415195 2024-11-24T02:46:55,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415195 2024-11-24T02:46:55,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415195) roll requested 2024-11-24T02:46:55,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415202 2024-11-24T02:46:55,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415202 2024-11-24T02:46:55,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415202) roll requested 2024-11-24T02:46:55,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415208 2024-11-24T02:46:55,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415202 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415208 2024-11-24T02:46:55,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415208) roll requested 2024-11-24T02:46:55,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415216 2024-11-24T02:46:55,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415216 2024-11-24T02:46:55,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,224 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415216) roll requested 2024-11-24T02:46:55,225 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415225 2024-11-24T02:46:55,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415225 2024-11-24T02:46:55,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415225) roll requested 2024-11-24T02:46:55,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415231 2024-11-24T02:46:55,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415225 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415231 2024-11-24T02:46:55,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415231) roll requested 2024-11-24T02:46:55,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415238 2024-11-24T02:46:55,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415238 2024-11-24T02:46:55,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415238) roll requested 2024-11-24T02:46:55,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415253 2024-11-24T02:46:55,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415253 2024-11-24T02:46:55,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415253) roll requested 2024-11-24T02:46:55,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415272 2024-11-24T02:46:55,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415272 2024-11-24T02:46:55,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,282 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415272) roll requested 2024-11-24T02:46:55,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415282 2024-11-24T02:46:55,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415282 2024-11-24T02:46:55,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415282) roll requested 2024-11-24T02:46:55,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415301 2024-11-24T02:46:55,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415282 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415301 2024-11-24T02:46:55,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415301) roll requested 2024-11-24T02:46:55,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415324 2024-11-24T02:46:55,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,346 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415324 2024-11-24T02:46:55,347 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,347 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,347 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415324) roll requested 2024-11-24T02:46:55,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415347 2024-11-24T02:46:55,368 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,368 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415324 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415347 2024-11-24T02:46:55,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415347) roll requested 2024-11-24T02:46:55,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415378 2024-11-24T02:46:55,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,389 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,389 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,389 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,389 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415347 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415378 2024-11-24T02:46:55,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415378) roll requested 2024-11-24T02:46:55,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415393 2024-11-24T02:46:55,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415393 2024-11-24T02:46:55,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415393) roll requested 2024-11-24T02:46:55,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415425 2024-11-24T02:46:55,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415425 2024-11-24T02:46:55,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415425) roll requested 2024-11-24T02:46:55,457 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415457 2024-11-24T02:46:55,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415457 2024-11-24T02:46:55,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,477 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415457) roll requested 2024-11-24T02:46:55,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415478 2024-11-24T02:46:55,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,490 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415478 2024-11-24T02:46:55,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415478) roll requested 2024-11-24T02:46:55,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415497 2024-11-24T02:46:55,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415497 2024-11-24T02:46:55,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,540 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415497) roll requested 2024-11-24T02:46:55,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415541 2024-11-24T02:46:55,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415541 2024-11-24T02:46:55,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415541) roll requested 2024-11-24T02:46:55,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415569 2024-11-24T02:46:55,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415569 2024-11-24T02:46:55,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415569) roll requested 2024-11-24T02:46:55,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415593 2024-11-24T02:46:55,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415569 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415593 2024-11-24T02:46:55,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415593) roll requested 2024-11-24T02:46:55,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415657 2024-11-24T02:46:55,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415593 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415657 2024-11-24T02:46:55,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,705 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415657) roll requested 2024-11-24T02:46:55,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415705 2024-11-24T02:46:55,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415705 2024-11-24T02:46:55,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415705) roll requested 2024-11-24T02:46:55,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415742 2024-11-24T02:46:55,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415742 2024-11-24T02:46:55,762 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,762 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415742) roll requested 2024-11-24T02:46:55,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415763 2024-11-24T02:46:55,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415763 2024-11-24T02:46:55,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,799 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415763) roll requested 2024-11-24T02:46:55,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415800 2024-11-24T02:46:55,816 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415800 2024-11-24T02:46:55,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415800) roll requested 2024-11-24T02:46:55,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415824 2024-11-24T02:46:55,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415824 2024-11-24T02:46:55,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,847 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415824) roll requested 2024-11-24T02:46:55,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415847 2024-11-24T02:46:55,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415847 2024-11-24T02:46:55,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,864 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415847) roll requested 2024-11-24T02:46:55,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415864 2024-11-24T02:46:55,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415847 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415864 2024-11-24T02:46:55,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415864) roll requested 2024-11-24T02:46:55,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415897 2024-11-24T02:46:55,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415897 2024-11-24T02:46:55,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,918 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415897) roll requested 2024-11-24T02:46:55,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415918 2024-11-24T02:46:55,949 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,949 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,949 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415918 2024-11-24T02:46:55,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415918) roll requested 2024-11-24T02:46:55,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415953 2024-11-24T02:46:55,961 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,961 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415953 2024-11-24T02:46:55,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415953) roll requested 2024-11-24T02:46:55,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415968 2024-11-24T02:46:55,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415968 2024-11-24T02:46:55,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:55,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:55,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415968) roll requested 2024-11-24T02:46:55,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416415982 2024-11-24T02:46:55,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:55,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415982 2024-11-24T02:46:56,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,001 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416415982) roll requested 2024-11-24T02:46:56,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416001 2024-11-24T02:46:56,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416415982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416001 2024-11-24T02:46:56,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416001) roll requested 2024-11-24T02:46:56,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416028 2024-11-24T02:46:56,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,054 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416028 2024-11-24T02:46:56,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416028) roll requested 2024-11-24T02:46:56,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416065 2024-11-24T02:46:56,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416065 2024-11-24T02:46:56,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,089 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416065) roll requested 2024-11-24T02:46:56,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416089 2024-11-24T02:46:56,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416089 2024-11-24T02:46:56,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,104 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416089) roll requested 2024-11-24T02:46:56,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416104 2024-11-24T02:46:56,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416104 2024-11-24T02:46:56,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416104) roll requested 2024-11-24T02:46:56,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416117 2024-11-24T02:46:56,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416104 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416117 2024-11-24T02:46:56,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416117) roll requested 2024-11-24T02:46:56,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416137 2024-11-24T02:46:56,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416137 2024-11-24T02:46:56,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416137) roll requested 2024-11-24T02:46:56,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416153 2024-11-24T02:46:56,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416153 2024-11-24T02:46:56,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,163 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416153) roll requested 2024-11-24T02:46:56,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416163 2024-11-24T02:46:56,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416163 2024-11-24T02:46:56,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416163) roll requested 2024-11-24T02:46:56,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416177 2024-11-24T02:46:56,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416163 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416177 2024-11-24T02:46:56,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416177) roll requested 2024-11-24T02:46:56,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416190 2024-11-24T02:46:56,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416177 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416190 2024-11-24T02:46:56,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416190) roll requested 2024-11-24T02:46:56,201 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416201 2024-11-24T02:46:56,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416190 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416201 2024-11-24T02:46:56,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,213 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416201) roll requested 2024-11-24T02:46:56,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416213 2024-11-24T02:46:56,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416213 2024-11-24T02:46:56,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416213) roll requested 2024-11-24T02:46:56,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416224 2024-11-24T02:46:56,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416213 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416224 2024-11-24T02:46:56,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416224) roll requested 2024-11-24T02:46:56,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416231 2024-11-24T02:46:56,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,240 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,240 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,240 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,240 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416231 2024-11-24T02:46:56,241 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,241 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,241 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416231) roll requested 2024-11-24T02:46:56,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416241 2024-11-24T02:46:56,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416231 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416241 2024-11-24T02:46:56,247 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,247 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416241) roll requested 2024-11-24T02:46:56,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416248 2024-11-24T02:46:56,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416241 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416248 2024-11-24T02:46:56,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416248) roll requested 2024-11-24T02:46:56,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416258 2024-11-24T02:46:56,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416258 2024-11-24T02:46:56,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416258) roll requested 2024-11-24T02:46:56,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416264 2024-11-24T02:46:56,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416258 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416264 2024-11-24T02:46:56,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416264) roll requested 2024-11-24T02:46:56,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416276 2024-11-24T02:46:56,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416276 2024-11-24T02:46:56,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416276) roll requested 2024-11-24T02:46:56,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416284 2024-11-24T02:46:56,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416284 2024-11-24T02:46:56,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416284) roll requested 2024-11-24T02:46:56,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416299 2024-11-24T02:46:56,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416299 2024-11-24T02:46:56,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416299) roll requested 2024-11-24T02:46:56,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416312 2024-11-24T02:46:56,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416312 2024-11-24T02:46:56,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416312) roll requested 2024-11-24T02:46:56,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416322 2024-11-24T02:46:56,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416322 2024-11-24T02:46:56,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416322) roll requested 2024-11-24T02:46:56,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416335 2024-11-24T02:46:56,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416335 2024-11-24T02:46:56,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416335) roll requested 2024-11-24T02:46:56,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416349 2024-11-24T02:46:56,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416349 2024-11-24T02:46:56,376 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,377 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,377 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,377 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416349) roll requested 2024-11-24T02:46:56,377 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416377 2024-11-24T02:46:56,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416377 2024-11-24T02:46:56,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416377) roll requested 2024-11-24T02:46:56,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416393 2024-11-24T02:46:56,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416377 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416393 2024-11-24T02:46:56,401 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,401 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,401 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,401 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416393) roll requested 2024-11-24T02:46:56,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416401 2024-11-24T02:46:56,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416401 2024-11-24T02:46:56,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416401) roll requested 2024-11-24T02:46:56,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416408 2024-11-24T02:46:56,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416401 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416408 2024-11-24T02:46:56,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416408) roll requested 2024-11-24T02:46:56,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416417 2024-11-24T02:46:56,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416417 2024-11-24T02:46:56,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416417) roll requested 2024-11-24T02:46:56,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416428 2024-11-24T02:46:56,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416428 2024-11-24T02:46:56,438 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,438 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,438 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416428) roll requested 2024-11-24T02:46:56,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416439 2024-11-24T02:46:56,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416439 2024-11-24T02:46:56,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416439) roll requested 2024-11-24T02:46:56,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416453 2024-11-24T02:46:56,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416453 2024-11-24T02:46:56,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416453) roll requested 2024-11-24T02:46:56,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416466 2024-11-24T02:46:56,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416466 2024-11-24T02:46:56,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416466) roll requested 2024-11-24T02:46:56,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416472 2024-11-24T02:46:56,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416472 2024-11-24T02:46:56,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,488 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,488 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416472) roll requested 2024-11-24T02:46:56,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416488 2024-11-24T02:46:56,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416488 2024-11-24T02:46:56,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416488) roll requested 2024-11-24T02:46:56,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416509 2024-11-24T02:46:56,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416488 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416509 2024-11-24T02:46:56,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416509) roll requested 2024-11-24T02:46:56,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416521 2024-11-24T02:46:56,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416509 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416521 2024-11-24T02:46:56,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416521) roll requested 2024-11-24T02:46:56,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416531 2024-11-24T02:46:56,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,539 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416531 2024-11-24T02:46:56,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,540 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416531) roll requested 2024-11-24T02:46:56,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416541 2024-11-24T02:46:56,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416541 2024-11-24T02:46:56,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416541) roll requested 2024-11-24T02:46:56,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416568 2024-11-24T02:46:56,580 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416568 2024-11-24T02:46:56,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416568) roll requested 2024-11-24T02:46:56,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416581 2024-11-24T02:46:56,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416581 2024-11-24T02:46:56,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416581) roll requested 2024-11-24T02:46:56,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416597 2024-11-24T02:46:56,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416597 2024-11-24T02:46:56,604 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,604 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416597) roll requested 2024-11-24T02:46:56,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416605 2024-11-24T02:46:56,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416605 2024-11-24T02:46:56,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,621 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416605) roll requested 2024-11-24T02:46:56,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416622 2024-11-24T02:46:56,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416622 2024-11-24T02:46:56,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,632 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,632 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416622) roll requested 2024-11-24T02:46:56,632 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416632 2024-11-24T02:46:56,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416622 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416632 2024-11-24T02:46:56,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,642 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,642 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,642 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416632) roll requested 2024-11-24T02:46:56,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416643 2024-11-24T02:46:56,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416632 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416643 2024-11-24T02:46:56,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416643) roll requested 2024-11-24T02:46:56,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416667 2024-11-24T02:46:56,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416667 2024-11-24T02:46:56,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,686 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416667) roll requested 2024-11-24T02:46:56,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416686 2024-11-24T02:46:56,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416686 2024-11-24T02:46:56,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416686) roll requested 2024-11-24T02:46:56,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416700 2024-11-24T02:46:56,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416700 2024-11-24T02:46:56,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,720 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416700) roll requested 2024-11-24T02:46:56,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416721 2024-11-24T02:46:56,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416721 2024-11-24T02:46:56,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,739 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416721) roll requested 2024-11-24T02:46:56,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416739 2024-11-24T02:46:56,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416739 2024-11-24T02:46:56,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416739) roll requested 2024-11-24T02:46:56,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416750 2024-11-24T02:46:56,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416750 2024-11-24T02:46:56,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416750) roll requested 2024-11-24T02:46:56,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416768 2024-11-24T02:46:56,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416768 2024-11-24T02:46:56,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416768) roll requested 2024-11-24T02:46:56,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416784 2024-11-24T02:46:56,791 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416784 2024-11-24T02:46:56,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416784) roll requested 2024-11-24T02:46:56,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416793 2024-11-24T02:46:56,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416793 2024-11-24T02:46:56,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416793) roll requested 2024-11-24T02:46:56,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416804 2024-11-24T02:46:56,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416804 2024-11-24T02:46:56,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416804) roll requested 2024-11-24T02:46:56,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416820 2024-11-24T02:46:56,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416820 2024-11-24T02:46:56,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416820) roll requested 2024-11-24T02:46:56,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416836 2024-11-24T02:46:56,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416836 2024-11-24T02:46:56,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416836) roll requested 2024-11-24T02:46:56,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416859 2024-11-24T02:46:56,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416859 2024-11-24T02:46:56,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416859) roll requested 2024-11-24T02:46:56,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416869 2024-11-24T02:46:56,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416869 2024-11-24T02:46:56,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416869) roll requested 2024-11-24T02:46:56,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416884 2024-11-24T02:46:56,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,896 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416884 2024-11-24T02:46:56,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,896 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416869 is not closed yet, will try archiving it next time 2024-11-24T02:46:56,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416884) roll requested 2024-11-24T02:46:56,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416897 2024-11-24T02:46:56,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416884 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416897 2024-11-24T02:46:56,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416897) roll requested 2024-11-24T02:46:56,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416912 2024-11-24T02:46:56,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,921 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416897 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416912 2024-11-24T02:46:56,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,925 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416912) roll requested 2024-11-24T02:46:56,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416926 2024-11-24T02:46:56,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416926 2024-11-24T02:46:56,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416926) roll requested 2024-11-24T02:46:56,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416948 2024-11-24T02:46:56,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416926 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416948 2024-11-24T02:46:56,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416948) roll requested 2024-11-24T02:46:56,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416970 2024-11-24T02:46:56,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,980 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416948 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416970 2024-11-24T02:46:56,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,983 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416970) roll requested 2024-11-24T02:46:56,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416983 2024-11-24T02:46:56,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:56,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416970 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416983 2024-11-24T02:46:56,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:56,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:56,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416983) roll requested 2024-11-24T02:46:56,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416416998 2024-11-24T02:46:57,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416983 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416998 2024-11-24T02:46:57,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416416998) roll requested 2024-11-24T02:46:57,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417006 2024-11-24T02:46:57,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416416998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417006 2024-11-24T02:46:57,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417006) roll requested 2024-11-24T02:46:57,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417018 2024-11-24T02:46:57,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417018 2024-11-24T02:46:57,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417018) roll requested 2024-11-24T02:46:57,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417062 2024-11-24T02:46:57,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417018 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417062 2024-11-24T02:46:57,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,086 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417062) roll requested 2024-11-24T02:46:57,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417086 2024-11-24T02:46:57,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417086 2024-11-24T02:46:57,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417086) roll requested 2024-11-24T02:46:57,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417116 2024-11-24T02:46:57,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417116 2024-11-24T02:46:57,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,150 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417116) roll requested 2024-11-24T02:46:57,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417150 2024-11-24T02:46:57,162 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,162 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,162 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,162 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417150 2024-11-24T02:46:57,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,171 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417150) roll requested 2024-11-24T02:46:57,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417171 2024-11-24T02:46:57,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417150 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417171 2024-11-24T02:46:57,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,191 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417171) roll requested 2024-11-24T02:46:57,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417191 2024-11-24T02:46:57,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417171 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417191 2024-11-24T02:46:57,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,204 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,204 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417191) roll requested 2024-11-24T02:46:57,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417204 2024-11-24T02:46:57,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417191 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417204 2024-11-24T02:46:57,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,223 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417204) roll requested 2024-11-24T02:46:57,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417224 2024-11-24T02:46:57,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,240 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417224 2024-11-24T02:46:57,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,242 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417224) roll requested 2024-11-24T02:46:57,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417242 2024-11-24T02:46:57,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417224 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417242 2024-11-24T02:46:57,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417242) roll requested 2024-11-24T02:46:57,254 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417253 2024-11-24T02:46:57,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,260 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,260 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417242 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417253 2024-11-24T02:46:57,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417253) roll requested 2024-11-24T02:46:57,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417261 2024-11-24T02:46:57,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417261 2024-11-24T02:46:57,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417261) roll requested 2024-11-24T02:46:57,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417287 2024-11-24T02:46:57,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417287 2024-11-24T02:46:57,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417287) roll requested 2024-11-24T02:46:57,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417301 2024-11-24T02:46:57,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417301 2024-11-24T02:46:57,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417287 is not closed yet, will try archiving it next time 2024-11-24T02:46:57,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417301) roll requested 2024-11-24T02:46:57,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417321 2024-11-24T02:46:57,382 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,382 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417321 2024-11-24T02:46:57,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417321) roll requested 2024-11-24T02:46:57,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417384 2024-11-24T02:46:57,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,397 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417321 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417384 2024-11-24T02:46:57,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417384) roll requested 2024-11-24T02:46:57,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417421 2024-11-24T02:46:57,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417384 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417421 2024-11-24T02:46:57,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417421) roll requested 2024-11-24T02:46:57,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417476 2024-11-24T02:46:57,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417421 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417476 2024-11-24T02:46:57,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,496 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417476) roll requested 2024-11-24T02:46:57,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417496 2024-11-24T02:46:57,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417496 2024-11-24T02:46:57,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417496) roll requested 2024-11-24T02:46:57,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417513 2024-11-24T02:46:57,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417496 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417513 2024-11-24T02:46:57,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417513) roll requested 2024-11-24T02:46:57,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417524 2024-11-24T02:46:57,534 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,534 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,534 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,534 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,534 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417524 2024-11-24T02:46:57,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417524) roll requested 2024-11-24T02:46:57,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417535 2024-11-24T02:46:57,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417524 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417535 2024-11-24T02:46:57,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,542 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,542 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417535) roll requested 2024-11-24T02:46:57,542 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417542 2024-11-24T02:46:57,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417535 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417542 2024-11-24T02:46:57,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417542) roll requested 2024-11-24T02:46:57,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417549 2024-11-24T02:46:57,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417542 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417549 2024-11-24T02:46:57,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,559 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417549) roll requested 2024-11-24T02:46:57,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417560 2024-11-24T02:46:57,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417560 2024-11-24T02:46:57,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417560) roll requested 2024-11-24T02:46:57,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417570 2024-11-24T02:46:57,576 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417570 2024-11-24T02:46:57,581 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,581 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,582 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417570) roll requested 2024-11-24T02:46:57,582 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417582 2024-11-24T02:46:57,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417582 2024-11-24T02:46:57,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417582) roll requested 2024-11-24T02:46:57,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417592 2024-11-24T02:46:57,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417582 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417592 2024-11-24T02:46:57,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417592) roll requested 2024-11-24T02:46:57,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417601 2024-11-24T02:46:57,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417601 2024-11-24T02:46:57,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417601) roll requested 2024-11-24T02:46:57,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417607 2024-11-24T02:46:57,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417607 2024-11-24T02:46:57,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417607) roll requested 2024-11-24T02:46:57,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417616 2024-11-24T02:46:57,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,622 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417607 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417616 2024-11-24T02:46:57,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417616) roll requested 2024-11-24T02:46:57,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417623 2024-11-24T02:46:57,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417623 2024-11-24T02:46:57,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417623) roll requested 2024-11-24T02:46:57,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417630 2024-11-24T02:46:57,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417630 2024-11-24T02:46:57,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417630) roll requested 2024-11-24T02:46:57,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417636 2024-11-24T02:46:57,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,649 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,649 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417630 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417636 2024-11-24T02:46:57,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417636) roll requested 2024-11-24T02:46:57,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417649 2024-11-24T02:46:57,656 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,656 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,656 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417649 2024-11-24T02:46:57,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417649) roll requested 2024-11-24T02:46:57,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417662 2024-11-24T02:46:57,681 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,681 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,681 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,681 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417662 2024-11-24T02:46:57,682 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,682 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,682 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,682 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417662) roll requested 2024-11-24T02:46:57,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417682 2024-11-24T02:46:57,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417682 2024-11-24T02:46:57,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417682) roll requested 2024-11-24T02:46:57,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417700 2024-11-24T02:46:57,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417682 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417700 2024-11-24T02:46:57,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417700) roll requested 2024-11-24T02:46:57,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417717 2024-11-24T02:46:57,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417717 2024-11-24T02:46:57,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,723 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417717) roll requested 2024-11-24T02:46:57,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417724 2024-11-24T02:46:57,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417724 2024-11-24T02:46:57,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417724) roll requested 2024-11-24T02:46:57,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417731 2024-11-24T02:46:57,740 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,740 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,740 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417731 2024-11-24T02:46:57,741 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,741 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417731) roll requested 2024-11-24T02:46:57,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417742 2024-11-24T02:46:57,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417731 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417742 2024-11-24T02:46:57,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417742) roll requested 2024-11-24T02:46:57,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417749 2024-11-24T02:46:57,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417742 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417749 2024-11-24T02:46:57,755 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,755 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417749) roll requested 2024-11-24T02:46:57,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417756 2024-11-24T02:46:57,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,761 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,761 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,761 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,761 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,761 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417749 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417756 2024-11-24T02:46:57,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,761 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,762 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417756) roll requested 2024-11-24T02:46:57,762 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417762 2024-11-24T02:46:57,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,767 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417762 2024-11-24T02:46:57,768 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,768 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,768 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417762) roll requested 2024-11-24T02:46:57,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417768 2024-11-24T02:46:57,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417762 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417768 2024-11-24T02:46:57,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417768) roll requested 2024-11-24T02:46:57,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417776 2024-11-24T02:46:57,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417768 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417776 2024-11-24T02:46:57,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417776) roll requested 2024-11-24T02:46:57,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417791 2024-11-24T02:46:57,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,797 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,797 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417791 2024-11-24T02:46:57,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417791) roll requested 2024-11-24T02:46:57,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417802 2024-11-24T02:46:57,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,808 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,808 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417802 2024-11-24T02:46:57,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417802) roll requested 2024-11-24T02:46:57,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417812 2024-11-24T02:46:57,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417812 2024-11-24T02:46:57,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417812) roll requested 2024-11-24T02:46:57,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417821 2024-11-24T02:46:57,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417821 2024-11-24T02:46:57,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417821) roll requested 2024-11-24T02:46:57,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417829 2024-11-24T02:46:57,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417829 2024-11-24T02:46:57,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417829) roll requested 2024-11-24T02:46:57,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417835 2024-11-24T02:46:57,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417835 2024-11-24T02:46:57,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,843 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417835) roll requested 2024-11-24T02:46:57,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417843 2024-11-24T02:46:57,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417843 2024-11-24T02:46:57,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,851 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417843) roll requested 2024-11-24T02:46:57,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417852 2024-11-24T02:46:57,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417843 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417852 2024-11-24T02:46:57,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417852) roll requested 2024-11-24T02:46:57,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417860 2024-11-24T02:46:57,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417852 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417860 2024-11-24T02:46:57,909 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,909 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417860) roll requested 2024-11-24T02:46:57,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417910 2024-11-24T02:46:57,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,928 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,928 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,928 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,928 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417910 2024-11-24T02:46:57,929 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417910) roll requested 2024-11-24T02:46:57,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417930 2024-11-24T02:46:57,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:57,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417930 2024-11-24T02:46:57,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:57,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:57,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417930) roll requested 2024-11-24T02:46:57,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416417952 2024-11-24T02:46:58,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417952 2024-11-24T02:46:58,015 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,015 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416417952) roll requested 2024-11-24T02:46:58,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418016 2024-11-24T02:46:58,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416417952 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418016 2024-11-24T02:46:58,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,027 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418016) roll requested 2024-11-24T02:46:58,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418029 2024-11-24T02:46:58,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418029 2024-11-24T02:46:58,048 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,048 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418029) roll requested 2024-11-24T02:46:58,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418049 2024-11-24T02:46:58,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418049 2024-11-24T02:46:58,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,065 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418049) roll requested 2024-11-24T02:46:58,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418065 2024-11-24T02:46:58,075 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418049 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418065 2024-11-24T02:46:58,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418065) roll requested 2024-11-24T02:46:58,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418097 2024-11-24T02:46:58,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418065 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418097 2024-11-24T02:46:58,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418097) roll requested 2024-11-24T02:46:58,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418117 2024-11-24T02:46:58,131 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418097 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418117 2024-11-24T02:46:58,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418117) roll requested 2024-11-24T02:46:58,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418141 2024-11-24T02:46:58,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,151 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418141 2024-11-24T02:46:58,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,152 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,152 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418141) roll requested 2024-11-24T02:46:58,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418152 2024-11-24T02:46:58,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418152 2024-11-24T02:46:58,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,164 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418152) roll requested 2024-11-24T02:46:58,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418165 2024-11-24T02:46:58,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418152 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418165 2024-11-24T02:46:58,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418165) roll requested 2024-11-24T02:46:58,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418174 2024-11-24T02:46:58,181 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,182 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418174 2024-11-24T02:46:58,182 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,182 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,182 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,182 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418174) roll requested 2024-11-24T02:46:58,183 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418183 2024-11-24T02:46:58,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418183 2024-11-24T02:46:58,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418183) roll requested 2024-11-24T02:46:58,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418197 2024-11-24T02:46:58,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418183 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418197 2024-11-24T02:46:58,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418197) roll requested 2024-11-24T02:46:58,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418220 2024-11-24T02:46:58,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418220 2024-11-24T02:46:58,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418220) roll requested 2024-11-24T02:46:58,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418244 2024-11-24T02:46:58,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418244 2024-11-24T02:46:58,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418244) roll requested 2024-11-24T02:46:58,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418264 2024-11-24T02:46:58,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418264 2024-11-24T02:46:58,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418264) roll requested 2024-11-24T02:46:58,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418279 2024-11-24T02:46:58,288 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,288 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418279 2024-11-24T02:46:58,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418279) roll requested 2024-11-24T02:46:58,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418298 2024-11-24T02:46:58,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,309 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,309 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,309 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,309 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418298 2024-11-24T02:46:58,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,309 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,310 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,310 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418298) roll requested 2024-11-24T02:46:58,310 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418310 2024-11-24T02:46:58,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418298 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418310 2024-11-24T02:46:58,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,318 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418310) roll requested 2024-11-24T02:46:58,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418318 2024-11-24T02:46:58,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418310 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418318 2024-11-24T02:46:58,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418318) roll requested 2024-11-24T02:46:58,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418328 2024-11-24T02:46:58,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418318 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418328 2024-11-24T02:46:58,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418328) roll requested 2024-11-24T02:46:58,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418336 2024-11-24T02:46:58,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,343 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,343 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418336 2024-11-24T02:46:58,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,344 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418336) roll requested 2024-11-24T02:46:58,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418345 2024-11-24T02:46:58,350 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,350 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,350 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418345 2024-11-24T02:46:58,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,352 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,352 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,352 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418345) roll requested 2024-11-24T02:46:58,352 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418352 2024-11-24T02:46:58,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418345 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418352 2024-11-24T02:46:58,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,359 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418352) roll requested 2024-11-24T02:46:58,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418359 2024-11-24T02:46:58,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418352 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418359 2024-11-24T02:46:58,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,367 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418359) roll requested 2024-11-24T02:46:58,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418367 2024-11-24T02:46:58,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418359 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418367 2024-11-24T02:46:58,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418367) roll requested 2024-11-24T02:46:58,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418381 2024-11-24T02:46:58,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418367 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418381 2024-11-24T02:46:58,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,389 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418381) roll requested 2024-11-24T02:46:58,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418390 2024-11-24T02:46:58,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418381 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418390 2024-11-24T02:46:58,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,397 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418390) roll requested 2024-11-24T02:46:58,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418397 2024-11-24T02:46:58,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418397 2024-11-24T02:46:58,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,406 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418397) roll requested 2024-11-24T02:46:58,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418407 2024-11-24T02:46:58,412 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,412 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418397 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418407 2024-11-24T02:46:58,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418407) roll requested 2024-11-24T02:46:58,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418417 2024-11-24T02:46:58,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,426 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,426 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418417 2024-11-24T02:46:58,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,428 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418417) roll requested 2024-11-24T02:46:58,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418428 2024-11-24T02:46:58,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,434 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,434 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,434 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418428 2024-11-24T02:46:58,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,435 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418428) roll requested 2024-11-24T02:46:58,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418435 2024-11-24T02:46:58,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418428 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418435 2024-11-24T02:46:58,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418435) roll requested 2024-11-24T02:46:58,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418456 2024-11-24T02:46:58,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418435 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418456 2024-11-24T02:46:58,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418456) roll requested 2024-11-24T02:46:58,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418476 2024-11-24T02:46:58,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418476 2024-11-24T02:46:58,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,502 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418476) roll requested 2024-11-24T02:46:58,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418502 2024-11-24T02:46:58,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,521 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,521 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,521 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418502 2024-11-24T02:46:58,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,524 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418502) roll requested 2024-11-24T02:46:58,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418525 2024-11-24T02:46:58,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,533 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418502 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418525 2024-11-24T02:46:58,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,533 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418525) roll requested 2024-11-24T02:46:58,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418533 2024-11-24T02:46:58,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418533 2024-11-24T02:46:58,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,549 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,549 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418533) roll requested 2024-11-24T02:46:58,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418549 2024-11-24T02:46:58,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418533 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418549 2024-11-24T02:46:58,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418549) roll requested 2024-11-24T02:46:58,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418563 2024-11-24T02:46:58,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418549 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418563 2024-11-24T02:46:58,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418563) roll requested 2024-11-24T02:46:58,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418580 2024-11-24T02:46:58,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418580 2024-11-24T02:46:58,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418580) roll requested 2024-11-24T02:46:58,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418587 2024-11-24T02:46:58,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418587 2024-11-24T02:46:58,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418587) roll requested 2024-11-24T02:46:58,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418597 2024-11-24T02:46:58,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418597 2024-11-24T02:46:58,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418597) roll requested 2024-11-24T02:46:58,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418609 2024-11-24T02:46:58,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418597 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418609 2024-11-24T02:46:58,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418609) roll requested 2024-11-24T02:46:58,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418620 2024-11-24T02:46:58,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418620 2024-11-24T02:46:58,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418620) roll requested 2024-11-24T02:46:58,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418629 2024-11-24T02:46:58,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418620 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418629 2024-11-24T02:46:58,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418629) roll requested 2024-11-24T02:46:58,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418637 2024-11-24T02:46:58,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418637 2024-11-24T02:46:58,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418637) roll requested 2024-11-24T02:46:58,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418655 2024-11-24T02:46:58,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,668 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,668 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418655 2024-11-24T02:46:58,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418655) roll requested 2024-11-24T02:46:58,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418669 2024-11-24T02:46:58,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418655 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418669 2024-11-24T02:46:58,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,675 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418669) roll requested 2024-11-24T02:46:58,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418676 2024-11-24T02:46:58,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418676 2024-11-24T02:46:58,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,692 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,692 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418676) roll requested 2024-11-24T02:46:58,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418692 2024-11-24T02:46:58,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418692 2024-11-24T02:46:58,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418692) roll requested 2024-11-24T02:46:58,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418706 2024-11-24T02:46:58,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418692 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418706 2024-11-24T02:46:58,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418706) roll requested 2024-11-24T02:46:58,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418716 2024-11-24T02:46:58,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,722 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418716 2024-11-24T02:46:58,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418716) roll requested 2024-11-24T02:46:58,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418725 2024-11-24T02:46:58,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,734 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418716 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418725 2024-11-24T02:46:58,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418725) roll requested 2024-11-24T02:46:58,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418735 2024-11-24T02:46:58,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418735 2024-11-24T02:46:58,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418735) roll requested 2024-11-24T02:46:58,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418748 2024-11-24T02:46:58,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418748 2024-11-24T02:46:58,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,757 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418748) roll requested 2024-11-24T02:46:58,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418758 2024-11-24T02:46:58,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418758 2024-11-24T02:46:58,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418758) roll requested 2024-11-24T02:46:58,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418765 2024-11-24T02:46:58,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418758 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418765 2024-11-24T02:46:58,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418765) roll requested 2024-11-24T02:46:58,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418784 2024-11-24T02:46:58,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418784 2024-11-24T02:46:58,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,803 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418784) roll requested 2024-11-24T02:46:58,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418804 2024-11-24T02:46:58,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418804 2024-11-24T02:46:58,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418804) roll requested 2024-11-24T02:46:58,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418823 2024-11-24T02:46:58,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,833 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418823 2024-11-24T02:46:58,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418823) roll requested 2024-11-24T02:46:58,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418838 2024-11-24T02:46:58,844 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,844 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,844 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418838 2024-11-24T02:46:58,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418838) roll requested 2024-11-24T02:46:58,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418848 2024-11-24T02:46:58,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,856 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,856 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418848 2024-11-24T02:46:58,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418848) roll requested 2024-11-24T02:46:58,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418860 2024-11-24T02:46:58,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418860 2024-11-24T02:46:58,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,884 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418860) roll requested 2024-11-24T02:46:58,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418885 2024-11-24T02:46:58,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418885 2024-11-24T02:46:58,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,920 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418885) roll requested 2024-11-24T02:46:58,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418921 2024-11-24T02:46:58,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:58,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418921 2024-11-24T02:46:58,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:58,956 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:58,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418921) roll requested 2024-11-24T02:46:58,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416418957 2024-11-24T02:46:59,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418957 2024-11-24T02:46:59,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416418957) roll requested 2024-11-24T02:46:59,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419013 2024-11-24T02:46:59,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416418957 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419013 2024-11-24T02:46:59,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,060 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419013) roll requested 2024-11-24T02:46:59,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419061 2024-11-24T02:46:59,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419061 2024-11-24T02:46:59,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419061) roll requested 2024-11-24T02:46:59,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419085 2024-11-24T02:46:59,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419085 2024-11-24T02:46:59,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419085) roll requested 2024-11-24T02:46:59,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419129 2024-11-24T02:46:59,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,141 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,141 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419085 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419129 2024-11-24T02:46:59,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419129) roll requested 2024-11-24T02:46:59,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419149 2024-11-24T02:46:59,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419149 2024-11-24T02:46:59,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419149) roll requested 2024-11-24T02:46:59,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419167 2024-11-24T02:46:59,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419167 2024-11-24T02:46:59,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,176 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,176 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419167) roll requested 2024-11-24T02:46:59,176 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419176 2024-11-24T02:46:59,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419176 2024-11-24T02:46:59,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419176) roll requested 2024-11-24T02:46:59,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419185 2024-11-24T02:46:59,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419176 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419185 2024-11-24T02:46:59,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,193 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,193 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419185) roll requested 2024-11-24T02:46:59,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419193 2024-11-24T02:46:59,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,207 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,207 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419193 2024-11-24T02:46:59,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,207 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419193) roll requested 2024-11-24T02:46:59,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419208 2024-11-24T02:46:59,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419193 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419208 2024-11-24T02:46:59,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419208) roll requested 2024-11-24T02:46:59,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419220 2024-11-24T02:46:59,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419220 2024-11-24T02:46:59,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419220) roll requested 2024-11-24T02:46:59,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419230 2024-11-24T02:46:59,236 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419230 2024-11-24T02:46:59,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419230) roll requested 2024-11-24T02:46:59,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419240 2024-11-24T02:46:59,247 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,247 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,247 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,247 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,247 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419240 2024-11-24T02:46:59,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419240) roll requested 2024-11-24T02:46:59,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419248 2024-11-24T02:46:59,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419248 2024-11-24T02:46:59,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,260 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,260 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419248) roll requested 2024-11-24T02:46:59,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419260 2024-11-24T02:46:59,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419260 2024-11-24T02:46:59,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,279 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419260) roll requested 2024-11-24T02:46:59,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419279 2024-11-24T02:46:59,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419260 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419279 2024-11-24T02:46:59,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419279) roll requested 2024-11-24T02:46:59,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419288 2024-11-24T02:46:59,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419279 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419288 2024-11-24T02:46:59,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419288) roll requested 2024-11-24T02:46:59,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419299 2024-11-24T02:46:59,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419299 2024-11-24T02:46:59,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419299) roll requested 2024-11-24T02:46:59,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419306 2024-11-24T02:46:59,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419306 2024-11-24T02:46:59,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,315 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419306) roll requested 2024-11-24T02:46:59,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419316 2024-11-24T02:46:59,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419316 2024-11-24T02:46:59,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419316) roll requested 2024-11-24T02:46:59,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419322 2024-11-24T02:46:59,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,329 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,329 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419316 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419322 2024-11-24T02:46:59,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,330 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,330 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,330 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419322) roll requested 2024-11-24T02:46:59,330 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419330 2024-11-24T02:46:59,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,336 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419330 2024-11-24T02:46:59,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,336 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,337 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419330) roll requested 2024-11-24T02:46:59,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419337 2024-11-24T02:46:59,345 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,345 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,345 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,345 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,345 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,345 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419330 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419337 2024-11-24T02:46:59,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419337) roll requested 2024-11-24T02:46:59,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419350 2024-11-24T02:46:59,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419337 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419350 2024-11-24T02:46:59,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419350) roll requested 2024-11-24T02:46:59,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419356 2024-11-24T02:46:59,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419356 2024-11-24T02:46:59,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419356) roll requested 2024-11-24T02:46:59,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419365 2024-11-24T02:46:59,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419365 2024-11-24T02:46:59,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,391 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,391 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419365) roll requested 2024-11-24T02:46:59,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419391 2024-11-24T02:46:59,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419391 2024-11-24T02:46:59,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419391) roll requested 2024-11-24T02:46:59,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419403 2024-11-24T02:46:59,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419391 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419403 2024-11-24T02:46:59,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,412 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419403) roll requested 2024-11-24T02:46:59,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419413 2024-11-24T02:46:59,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419413 2024-11-24T02:46:59,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419413) roll requested 2024-11-24T02:46:59,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419422 2024-11-24T02:46:59,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419422 2024-11-24T02:46:59,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419422) roll requested 2024-11-24T02:46:59,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419434 2024-11-24T02:46:59,440 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,440 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,440 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419434 2024-11-24T02:46:59,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419434) roll requested 2024-11-24T02:46:59,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419443 2024-11-24T02:46:59,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419443 2024-11-24T02:46:59,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419443) roll requested 2024-11-24T02:46:59,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419454 2024-11-24T02:46:59,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419454 2024-11-24T02:46:59,463 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,463 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,463 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,463 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419454) roll requested 2024-11-24T02:46:59,463 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419463 2024-11-24T02:46:59,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419454 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419463 2024-11-24T02:46:59,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,490 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419463) roll requested 2024-11-24T02:46:59,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419490 2024-11-24T02:46:59,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419463 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419490 2024-11-24T02:46:59,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,500 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419490) roll requested 2024-11-24T02:46:59,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419500 2024-11-24T02:46:59,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419490 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419500 2024-11-24T02:46:59,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,521 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419500) roll requested 2024-11-24T02:46:59,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419521 2024-11-24T02:46:59,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419521 2024-11-24T02:46:59,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,536 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419521) roll requested 2024-11-24T02:46:59,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419537 2024-11-24T02:46:59,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419537 2024-11-24T02:46:59,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,543 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419537) roll requested 2024-11-24T02:46:59,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419544 2024-11-24T02:46:59,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419544 2024-11-24T02:46:59,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,550 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419544) roll requested 2024-11-24T02:46:59,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419551 2024-11-24T02:46:59,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,556 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419551 2024-11-24T02:46:59,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419551) roll requested 2024-11-24T02:46:59,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419561 2024-11-24T02:46:59,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419551 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419561 2024-11-24T02:46:59,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419561) roll requested 2024-11-24T02:46:59,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419570 2024-11-24T02:46:59,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419561 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419570 2024-11-24T02:46:59,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419570) roll requested 2024-11-24T02:46:59,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419580 2024-11-24T02:46:59,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419580 2024-11-24T02:46:59,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419580) roll requested 2024-11-24T02:46:59,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419588 2024-11-24T02:46:59,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419588 2024-11-24T02:46:59,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419588) roll requested 2024-11-24T02:46:59,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419595 2024-11-24T02:46:59,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419595 2024-11-24T02:46:59,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,608 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419595) roll requested 2024-11-24T02:46:59,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419608 2024-11-24T02:46:59,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419595 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419608 2024-11-24T02:46:59,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,627 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419608) roll requested 2024-11-24T02:46:59,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419628 2024-11-24T02:46:59,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419628 2024-11-24T02:46:59,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419628) roll requested 2024-11-24T02:46:59,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419638 2024-11-24T02:46:59,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419638 2024-11-24T02:46:59,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419638) roll requested 2024-11-24T02:46:59,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419644 2024-11-24T02:46:59,650 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,651 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,651 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419644 2024-11-24T02:46:59,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419644) roll requested 2024-11-24T02:46:59,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419656 2024-11-24T02:46:59,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419656 2024-11-24T02:46:59,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419656) roll requested 2024-11-24T02:46:59,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419668 2024-11-24T02:46:59,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419668 2024-11-24T02:46:59,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419668) roll requested 2024-11-24T02:46:59,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419674 2024-11-24T02:46:59,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419674 2024-11-24T02:46:59,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419674) roll requested 2024-11-24T02:46:59,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419684 2024-11-24T02:46:59,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419684 2024-11-24T02:46:59,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419684) roll requested 2024-11-24T02:46:59,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419690 2024-11-24T02:46:59,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419690 2024-11-24T02:46:59,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419690) roll requested 2024-11-24T02:46:59,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419697 2024-11-24T02:46:59,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419697 2024-11-24T02:46:59,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419697) roll requested 2024-11-24T02:46:59,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419704 2024-11-24T02:46:59,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419697 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419704 2024-11-24T02:46:59,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,710 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419704) roll requested 2024-11-24T02:46:59,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419710 2024-11-24T02:46:59,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419704 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419710 2024-11-24T02:46:59,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419710) roll requested 2024-11-24T02:46:59,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419717 2024-11-24T02:46:59,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419717 2024-11-24T02:46:59,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,725 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419717) roll requested 2024-11-24T02:46:59,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419725 2024-11-24T02:46:59,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,732 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419725 2024-11-24T02:46:59,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419725) roll requested 2024-11-24T02:46:59,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419733 2024-11-24T02:46:59,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419733 2024-11-24T02:46:59,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419733) roll requested 2024-11-24T02:46:59,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419740 2024-11-24T02:46:59,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419733 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419740 2024-11-24T02:46:59,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419740) roll requested 2024-11-24T02:46:59,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419748 2024-11-24T02:46:59,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,755 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419748 2024-11-24T02:46:59,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419748) roll requested 2024-11-24T02:46:59,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419756 2024-11-24T02:46:59,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419756 2024-11-24T02:46:59,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419756) roll requested 2024-11-24T02:46:59,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419767 2024-11-24T02:46:59,773 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419767 2024-11-24T02:46:59,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,774 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419767) roll requested 2024-11-24T02:46:59,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419774 2024-11-24T02:46:59,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419774 2024-11-24T02:46:59,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419774) roll requested 2024-11-24T02:46:59,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419782 2024-11-24T02:46:59,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,791 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,791 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419774 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419782 2024-11-24T02:46:59,797 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,797 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419782) roll requested 2024-11-24T02:46:59,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419797 2024-11-24T02:46:59,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419797 2024-11-24T02:46:59,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419797) roll requested 2024-11-24T02:46:59,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419805 2024-11-24T02:46:59,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419797 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419805 2024-11-24T02:46:59,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419805) roll requested 2024-11-24T02:46:59,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419812 2024-11-24T02:46:59,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,818 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,818 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,818 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,818 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419812 2024-11-24T02:46:59,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,819 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,819 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419812) roll requested 2024-11-24T02:46:59,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419819 2024-11-24T02:46:59,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419819 2024-11-24T02:46:59,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,828 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419819) roll requested 2024-11-24T02:46:59,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419828 2024-11-24T02:46:59,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419819 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419828 2024-11-24T02:46:59,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,834 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419828) roll requested 2024-11-24T02:46:59,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419835 2024-11-24T02:46:59,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419828 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419835 2024-11-24T02:46:59,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419835) roll requested 2024-11-24T02:46:59,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419842 2024-11-24T02:46:59,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419842 2024-11-24T02:46:59,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419842) roll requested 2024-11-24T02:46:59,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419853 2024-11-24T02:46:59,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419853 2024-11-24T02:46:59,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,859 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,859 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419853) roll requested 2024-11-24T02:46:59,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419859 2024-11-24T02:46:59,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419859 2024-11-24T02:46:59,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419859) roll requested 2024-11-24T02:46:59,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419873 2024-11-24T02:46:59,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,902 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419859 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419873 2024-11-24T02:46:59,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419873) roll requested 2024-11-24T02:46:59,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419914 2024-11-24T02:46:59,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419873 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419914 2024-11-24T02:46:59,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419914) roll requested 2024-11-24T02:46:59,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419924 2024-11-24T02:46:59,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419924 2024-11-24T02:46:59,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419924) roll requested 2024-11-24T02:46:59,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419931 2024-11-24T02:46:59,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419931 2024-11-24T02:46:59,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419931) roll requested 2024-11-24T02:46:59,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419937 2024-11-24T02:46:59,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419937 2024-11-24T02:46:59,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419937) roll requested 2024-11-24T02:46:59,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419945 2024-11-24T02:46:59,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,952 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419945 2024-11-24T02:46:59,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419945) roll requested 2024-11-24T02:46:59,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419953 2024-11-24T02:46:59,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419953 2024-11-24T02:46:59,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419953) roll requested 2024-11-24T02:46:59,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419958 2024-11-24T02:46:59,967 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,967 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419958 2024-11-24T02:46:59,968 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,968 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,968 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,968 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419958) roll requested 2024-11-24T02:46:59,968 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419968 2024-11-24T02:46:59,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419968 2024-11-24T02:46:59,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419968) roll requested 2024-11-24T02:46:59,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419978 2024-11-24T02:46:59,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419968 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419978 2024-11-24T02:46:59,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419978) roll requested 2024-11-24T02:46:59,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419985 2024-11-24T02:46:59,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419985 2024-11-24T02:46:59,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,991 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,991 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419985) roll requested 2024-11-24T02:46:59,991 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419991 2024-11-24T02:46:59,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:46:59,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419991 2024-11-24T02:46:59,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:46:59,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:46:59,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419991) roll requested 2024-11-24T02:46:59,998 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416419998 2024-11-24T02:47:00,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419991 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419998 2024-11-24T02:47:00,003 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,003 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,003 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416419998) roll requested 2024-11-24T02:47:00,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420003 2024-11-24T02:47:00,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416419998 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420003 2024-11-24T02:47:00,009 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,009 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420003) roll requested 2024-11-24T02:47:00,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420010 2024-11-24T02:47:00,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,015 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420003 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420010 2024-11-24T02:47:00,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,016 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,016 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420010) roll requested 2024-11-24T02:47:00,016 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420016 2024-11-24T02:47:00,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,021 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,021 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420016 2024-11-24T02:47:00,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,022 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,022 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420016) roll requested 2024-11-24T02:47:00,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420023 2024-11-24T02:47:00,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420016 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420023 2024-11-24T02:47:00,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420023) roll requested 2024-11-24T02:47:00,039 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420039 2024-11-24T02:47:00,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,047 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,047 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420023 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420039 2024-11-24T02:47:00,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,048 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420039) roll requested 2024-11-24T02:47:00,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420048 2024-11-24T02:47:00,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,054 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420048 2024-11-24T02:47:00,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,055 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420048) roll requested 2024-11-24T02:47:00,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420055 2024-11-24T02:47:00,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420048 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420055 2024-11-24T02:47:00,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420055) roll requested 2024-11-24T02:47:00,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420061 2024-11-24T02:47:00,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420061 2024-11-24T02:47:00,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420061) roll requested 2024-11-24T02:47:00,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420069 2024-11-24T02:47:00,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,075 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,075 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420069 2024-11-24T02:47:00,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420069) roll requested 2024-11-24T02:47:00,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420076 2024-11-24T02:47:00,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420069 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420076 2024-11-24T02:47:00,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420076) roll requested 2024-11-24T02:47:00,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420084 2024-11-24T02:47:00,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420076 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420084 2024-11-24T02:47:00,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,090 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420084) roll requested 2024-11-24T02:47:00,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420090 2024-11-24T02:47:00,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420090 2024-11-24T02:47:00,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420090) roll requested 2024-11-24T02:47:00,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420096 2024-11-24T02:47:00,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420090 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420096 2024-11-24T02:47:00,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420096) roll requested 2024-11-24T02:47:00,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420103 2024-11-24T02:47:00,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420103 2024-11-24T02:47:00,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420103) roll requested 2024-11-24T02:47:00,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420113 2024-11-24T02:47:00,119 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,119 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,119 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420113 2024-11-24T02:47:00,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,120 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420113) roll requested 2024-11-24T02:47:00,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420120 2024-11-24T02:47:00,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,126 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,126 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420120 2024-11-24T02:47:00,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,127 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420120) roll requested 2024-11-24T02:47:00,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420128 2024-11-24T02:47:00,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420120 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420128 2024-11-24T02:47:00,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,133 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420128) roll requested 2024-11-24T02:47:00,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420133 2024-11-24T02:47:00,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420133 2024-11-24T02:47:00,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420133) roll requested 2024-11-24T02:47:00,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420145 2024-11-24T02:47:00,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420145 2024-11-24T02:47:00,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420145) roll requested 2024-11-24T02:47:00,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420153 2024-11-24T02:47:00,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,163 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420153 2024-11-24T02:47:00,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,164 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420153) roll requested 2024-11-24T02:47:00,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420165 2024-11-24T02:47:00,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420165 2024-11-24T02:47:00,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420165) roll requested 2024-11-24T02:47:00,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420175 2024-11-24T02:47:00,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420175 2024-11-24T02:47:00,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420175) roll requested 2024-11-24T02:47:00,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420184 2024-11-24T02:47:00,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420184 2024-11-24T02:47:00,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,192 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420184) roll requested 2024-11-24T02:47:00,193 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420192 2024-11-24T02:47:00,200 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,200 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,200 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420184 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420192 2024-11-24T02:47:00,201 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,201 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,201 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,201 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420192) roll requested 2024-11-24T02:47:00,202 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420201 2024-11-24T02:47:00,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420192 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420201 2024-11-24T02:47:00,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,222 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420201) roll requested 2024-11-24T02:47:00,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420222 2024-11-24T02:47:00,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420201 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420222 2024-11-24T02:47:00,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,230 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420222) roll requested 2024-11-24T02:47:00,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420230 2024-11-24T02:47:00,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420230 2024-11-24T02:47:00,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420230) roll requested 2024-11-24T02:47:00,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420240 2024-11-24T02:47:00,246 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,246 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,247 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420230 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420240 2024-11-24T02:47:00,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420240) roll requested 2024-11-24T02:47:00,248 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420248 2024-11-24T02:47:00,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420248 2024-11-24T02:47:00,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420248) roll requested 2024-11-24T02:47:00,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420269 2024-11-24T02:47:00,276 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420269 2024-11-24T02:47:00,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,285 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420269) roll requested 2024-11-24T02:47:00,286 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420286 2024-11-24T02:47:00,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420286 2024-11-24T02:47:00,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420286) roll requested 2024-11-24T02:47:00,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420306 2024-11-24T02:47:00,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420306 2024-11-24T02:47:00,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420306) roll requested 2024-11-24T02:47:00,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420314 2024-11-24T02:47:00,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420314 2024-11-24T02:47:00,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420314) roll requested 2024-11-24T02:47:00,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420363 2024-11-24T02:47:00,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420363 2024-11-24T02:47:00,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,390 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420363) roll requested 2024-11-24T02:47:00,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420390 2024-11-24T02:47:00,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420390 2024-11-24T02:47:00,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,422 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420390) roll requested 2024-11-24T02:47:00,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420422 2024-11-24T02:47:00,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420390 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420422 2024-11-24T02:47:00,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420422) roll requested 2024-11-24T02:47:00,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420436 2024-11-24T02:47:00,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420436 2024-11-24T02:47:00,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420436) roll requested 2024-11-24T02:47:00,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420445 2024-11-24T02:47:00,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,452 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,452 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420436 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420445 2024-11-24T02:47:00,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420445) roll requested 2024-11-24T02:47:00,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420453 2024-11-24T02:47:00,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420453 2024-11-24T02:47:00,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,461 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420453) roll requested 2024-11-24T02:47:00,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420461 2024-11-24T02:47:00,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420461 2024-11-24T02:47:00,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420461) roll requested 2024-11-24T02:47:00,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420473 2024-11-24T02:47:00,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420473 2024-11-24T02:47:00,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420473) roll requested 2024-11-24T02:47:00,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420480 2024-11-24T02:47:00,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420473 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420480 2024-11-24T02:47:00,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,486 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420480) roll requested 2024-11-24T02:47:00,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420487 2024-11-24T02:47:00,492 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,492 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,492 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420487 2024-11-24T02:47:00,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,493 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420487) roll requested 2024-11-24T02:47:00,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420493 2024-11-24T02:47:00,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420493 2024-11-24T02:47:00,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,504 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420493) roll requested 2024-11-24T02:47:00,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420505 2024-11-24T02:47:00,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420505 2024-11-24T02:47:00,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,512 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420505) roll requested 2024-11-24T02:47:00,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420513 2024-11-24T02:47:00,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,518 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420513 2024-11-24T02:47:00,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420513) roll requested 2024-11-24T02:47:00,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420521 2024-11-24T02:47:00,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,527 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420513 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420521 2024-11-24T02:47:00,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,528 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,529 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420521) roll requested 2024-11-24T02:47:00,529 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420529 2024-11-24T02:47:00,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420529 2024-11-24T02:47:00,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420529) roll requested 2024-11-24T02:47:00,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420545 2024-11-24T02:47:00,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420529 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420545 2024-11-24T02:47:00,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420545) roll requested 2024-11-24T02:47:00,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420560 2024-11-24T02:47:00,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420545 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420560 2024-11-24T02:47:00,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,569 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420560) roll requested 2024-11-24T02:47:00,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420570 2024-11-24T02:47:00,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420570 2024-11-24T02:47:00,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420570) roll requested 2024-11-24T02:47:00,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420580 2024-11-24T02:47:00,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420570 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420580 2024-11-24T02:47:00,586 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,586 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420580) roll requested 2024-11-24T02:47:00,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420587 2024-11-24T02:47:00,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420587 2024-11-24T02:47:00,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,594 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420587) roll requested 2024-11-24T02:47:00,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420594 2024-11-24T02:47:00,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420594 2024-11-24T02:47:00,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,600 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,600 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420594) roll requested 2024-11-24T02:47:00,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420601 2024-11-24T02:47:00,610 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,610 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,610 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420601 2024-11-24T02:47:00,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,620 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420601) roll requested 2024-11-24T02:47:00,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420621 2024-11-24T02:47:00,631 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,631 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,631 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,631 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420601 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420621 2024-11-24T02:47:00,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420621) roll requested 2024-11-24T02:47:00,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420633 2024-11-24T02:47:00,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420621 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420633 2024-11-24T02:47:00,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420633) roll requested 2024-11-24T02:47:00,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420640 2024-11-24T02:47:00,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420640 2024-11-24T02:47:00,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,649 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420640) roll requested 2024-11-24T02:47:00,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420649 2024-11-24T02:47:00,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420649 2024-11-24T02:47:00,657 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,657 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420649) roll requested 2024-11-24T02:47:00,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420657 2024-11-24T02:47:00,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,667 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,667 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,667 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,667 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420649 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420657 2024-11-24T02:47:00,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,667 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420657) roll requested 2024-11-24T02:47:00,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420668 2024-11-24T02:47:00,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420657 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420668 2024-11-24T02:47:00,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420668) roll requested 2024-11-24T02:47:00,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420684 2024-11-24T02:47:00,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420684 2024-11-24T02:47:00,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420684) roll requested 2024-11-24T02:47:00,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420712 2024-11-24T02:47:00,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420712 2024-11-24T02:47:00,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,719 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420712) roll requested 2024-11-24T02:47:00,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420719 2024-11-24T02:47:00,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420719 2024-11-24T02:47:00,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420719) roll requested 2024-11-24T02:47:00,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420730 2024-11-24T02:47:00,739 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,739 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420730 2024-11-24T02:47:00,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,741 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420730) roll requested 2024-11-24T02:47:00,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420741 2024-11-24T02:47:00,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420741 2024-11-24T02:47:00,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420741) roll requested 2024-11-24T02:47:00,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420752 2024-11-24T02:47:00,759 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420741 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420752 2024-11-24T02:47:00,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420752) roll requested 2024-11-24T02:47:00,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420761 2024-11-24T02:47:00,767 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420761 2024-11-24T02:47:00,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420761) roll requested 2024-11-24T02:47:00,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420777 2024-11-24T02:47:00,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420777 2024-11-24T02:47:00,785 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,785 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420777) roll requested 2024-11-24T02:47:00,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420786 2024-11-24T02:47:00,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420786 2024-11-24T02:47:00,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420786) roll requested 2024-11-24T02:47:00,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420796 2024-11-24T02:47:00,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420796 2024-11-24T02:47:00,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,804 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420796) roll requested 2024-11-24T02:47:00,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420804 2024-11-24T02:47:00,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420804 2024-11-24T02:47:00,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420804) roll requested 2024-11-24T02:47:00,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420814 2024-11-24T02:47:00,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420804 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420814 2024-11-24T02:47:00,821 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,821 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420814) roll requested 2024-11-24T02:47:00,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420821 2024-11-24T02:47:00,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,833 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,833 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420821 2024-11-24T02:47:00,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420821) roll requested 2024-11-24T02:47:00,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420837 2024-11-24T02:47:00,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420821 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420837 2024-11-24T02:47:00,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420837) roll requested 2024-11-24T02:47:00,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420845 2024-11-24T02:47:00,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,853 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,853 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,853 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,853 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,853 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420837 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420845 2024-11-24T02:47:00,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,854 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420845) roll requested 2024-11-24T02:47:00,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420854 2024-11-24T02:47:00,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420854 2024-11-24T02:47:00,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420854) roll requested 2024-11-24T02:47:00,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420862 2024-11-24T02:47:00,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420862 2024-11-24T02:47:00,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420862) roll requested 2024-11-24T02:47:00,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420875 2024-11-24T02:47:00,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420875 2024-11-24T02:47:00,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420875) roll requested 2024-11-24T02:47:00,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420883 2024-11-24T02:47:00,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420883 2024-11-24T02:47:00,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420883) roll requested 2024-11-24T02:47:00,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420891 2024-11-24T02:47:00,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420891 2024-11-24T02:47:00,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,904 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420891) roll requested 2024-11-24T02:47:00,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420904 2024-11-24T02:47:00,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,909 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,909 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,909 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420891 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420904 2024-11-24T02:47:00,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,910 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420904) roll requested 2024-11-24T02:47:00,910 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420910 2024-11-24T02:47:00,917 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420910 2024-11-24T02:47:00,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420910) roll requested 2024-11-24T02:47:00,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420921 2024-11-24T02:47:00,927 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,927 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,928 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420910 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420921 2024-11-24T02:47:00,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420921) roll requested 2024-11-24T02:47:00,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420928 2024-11-24T02:47:00,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420928 2024-11-24T02:47:00,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,936 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420928) roll requested 2024-11-24T02:47:00,936 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420936 2024-11-24T02:47:00,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420936 2024-11-24T02:47:00,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,943 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420936) roll requested 2024-11-24T02:47:00,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420943 2024-11-24T02:47:00,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420936 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420943 2024-11-24T02:47:00,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420943) roll requested 2024-11-24T02:47:00,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420949 2024-11-24T02:47:00,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420943 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420949 2024-11-24T02:47:00,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420949) roll requested 2024-11-24T02:47:00,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420969 2024-11-24T02:47:00,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420969 2024-11-24T02:47:00,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420969) roll requested 2024-11-24T02:47:00,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420992 2024-11-24T02:47:00,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:00,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420992 2024-11-24T02:47:00,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:00,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:00,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420992) roll requested 2024-11-24T02:47:01,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416420999 2024-11-24T02:47:01,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420999 2024-11-24T02:47:01,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,010 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416420999) roll requested 2024-11-24T02:47:01,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421010 2024-11-24T02:47:01,020 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416420999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421010 2024-11-24T02:47:01,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421010) roll requested 2024-11-24T02:47:01,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421028 2024-11-24T02:47:01,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421010 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421028 2024-11-24T02:47:01,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421028) roll requested 2024-11-24T02:47:01,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421037 2024-11-24T02:47:01,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421037 2024-11-24T02:47:01,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421037) roll requested 2024-11-24T02:47:01,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421045 2024-11-24T02:47:01,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421037 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421045 2024-11-24T02:47:01,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,058 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421045) roll requested 2024-11-24T02:47:01,058 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421058 2024-11-24T02:47:01,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421058 2024-11-24T02:47:01,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,074 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,074 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,074 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421058) roll requested 2024-11-24T02:47:01,074 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421074 2024-11-24T02:47:01,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421058 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421074 2024-11-24T02:47:01,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,084 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421074) roll requested 2024-11-24T02:47:01,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421084 2024-11-24T02:47:01,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421074 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421084 2024-11-24T02:47:01,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421084) roll requested 2024-11-24T02:47:01,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421096 2024-11-24T02:47:01,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421084 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421096 2024-11-24T02:47:01,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421096) roll requested 2024-11-24T02:47:01,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421106 2024-11-24T02:47:01,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,116 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421106 2024-11-24T02:47:01,117 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,117 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,117 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421106) roll requested 2024-11-24T02:47:01,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421117 2024-11-24T02:47:01,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,124 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421117 2024-11-24T02:47:01,125 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,125 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421117) roll requested 2024-11-24T02:47:01,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421126 2024-11-24T02:47:01,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421117 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421126 2024-11-24T02:47:01,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,137 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421126) roll requested 2024-11-24T02:47:01,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421138 2024-11-24T02:47:01,144 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421138 2024-11-24T02:47:01,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421138) roll requested 2024-11-24T02:47:01,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421145 2024-11-24T02:47:01,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421138 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421145 2024-11-24T02:47:01,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421145) roll requested 2024-11-24T02:47:01,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421153 2024-11-24T02:47:01,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421153 2024-11-24T02:47:01,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421153) roll requested 2024-11-24T02:47:01,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421162 2024-11-24T02:47:01,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421162 2024-11-24T02:47:01,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421162) roll requested 2024-11-24T02:47:01,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421173 2024-11-24T02:47:01,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,183 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421162 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421173 2024-11-24T02:47:01,184 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,184 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421173) roll requested 2024-11-24T02:47:01,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421185 2024-11-24T02:47:01,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421173 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421185 2024-11-24T02:47:01,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421185) roll requested 2024-11-24T02:47:01,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421196 2024-11-24T02:47:01,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,207 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,208 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421185 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421196 2024-11-24T02:47:01,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,215 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,215 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,215 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421196) roll requested 2024-11-24T02:47:01,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421216 2024-11-24T02:47:01,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421196 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421216 2024-11-24T02:47:01,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,231 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421216) roll requested 2024-11-24T02:47:01,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421232 2024-11-24T02:47:01,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421232 2024-11-24T02:47:01,243 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,243 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,243 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,243 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421232) roll requested 2024-11-24T02:47:01,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421244 2024-11-24T02:47:01,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421244 2024-11-24T02:47:01,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,257 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421244) roll requested 2024-11-24T02:47:01,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421257 2024-11-24T02:47:01,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421244 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421257 2024-11-24T02:47:01,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421257) roll requested 2024-11-24T02:47:01,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421277 2024-11-24T02:47:01,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,285 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,285 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421257 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421277 2024-11-24T02:47:01,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421277) roll requested 2024-11-24T02:47:01,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421288 2024-11-24T02:47:01,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421288 2024-11-24T02:47:01,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421288) roll requested 2024-11-24T02:47:01,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421297 2024-11-24T02:47:01,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421297 2024-11-24T02:47:01,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,308 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421297) roll requested 2024-11-24T02:47:01,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421308 2024-11-24T02:47:01,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,319 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,319 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,319 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421308 2024-11-24T02:47:01,319 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,319 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,319 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421308) roll requested 2024-11-24T02:47:01,320 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421320 2024-11-24T02:47:01,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421308 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421320 2024-11-24T02:47:01,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421320) roll requested 2024-11-24T02:47:01,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421328 2024-11-24T02:47:01,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421328 2024-11-24T02:47:01,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421328) roll requested 2024-11-24T02:47:01,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421335 2024-11-24T02:47:01,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421335 2024-11-24T02:47:01,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,341 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421335) roll requested 2024-11-24T02:47:01,342 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421342 2024-11-24T02:47:01,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421335 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421342 2024-11-24T02:47:01,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,357 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421342) roll requested 2024-11-24T02:47:01,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421357 2024-11-24T02:47:01,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,364 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,364 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,364 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421357 2024-11-24T02:47:01,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,365 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421357) roll requested 2024-11-24T02:47:01,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421365 2024-11-24T02:47:01,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,375 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,375 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421357 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421365 2024-11-24T02:47:01,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,375 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421365) roll requested 2024-11-24T02:47:01,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421375 2024-11-24T02:47:01,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421365 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421375 2024-11-24T02:47:01,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421375) roll requested 2024-11-24T02:47:01,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421388 2024-11-24T02:47:01,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,404 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421375 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421388 2024-11-24T02:47:01,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,407 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421388) roll requested 2024-11-24T02:47:01,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421407 2024-11-24T02:47:01,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,423 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,423 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,423 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421388 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421407 2024-11-24T02:47:01,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,425 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421407) roll requested 2024-11-24T02:47:01,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421425 2024-11-24T02:47:01,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,431 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421407 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421425 2024-11-24T02:47:01,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421425) roll requested 2024-11-24T02:47:01,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421432 2024-11-24T02:47:01,446 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,446 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,446 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,446 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,446 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,446 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421425 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421432 2024-11-24T02:47:01,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,446 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,447 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421432) roll requested 2024-11-24T02:47:01,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421447 2024-11-24T02:47:01,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421447 2024-11-24T02:47:01,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,460 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421447) roll requested 2024-11-24T02:47:01,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421461 2024-11-24T02:47:01,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421447 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421461 2024-11-24T02:47:01,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,468 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421461) roll requested 2024-11-24T02:47:01,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421468 2024-11-24T02:47:01,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421461 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421468 2024-11-24T02:47:01,474 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421468) roll requested 2024-11-24T02:47:01,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421475 2024-11-24T02:47:01,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421468 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421475 2024-11-24T02:47:01,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421475) roll requested 2024-11-24T02:47:01,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421481 2024-11-24T02:47:01,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421481 2024-11-24T02:47:01,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421481) roll requested 2024-11-24T02:47:01,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421487 2024-11-24T02:47:01,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421487 2024-11-24T02:47:01,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421487) roll requested 2024-11-24T02:47:01,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421497 2024-11-24T02:47:01,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421497 2024-11-24T02:47:01,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421497) roll requested 2024-11-24T02:47:01,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421510 2024-11-24T02:47:01,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421510 2024-11-24T02:47:01,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,516 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421510) roll requested 2024-11-24T02:47:01,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421517 2024-11-24T02:47:01,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421517 2024-11-24T02:47:01,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421517) roll requested 2024-11-24T02:47:01,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421523 2024-11-24T02:47:01,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421523 2024-11-24T02:47:01,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421523) roll requested 2024-11-24T02:47:01,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421531 2024-11-24T02:47:01,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421531 2024-11-24T02:47:01,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421531) roll requested 2024-11-24T02:47:01,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421537 2024-11-24T02:47:01,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421537 2024-11-24T02:47:01,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421537) roll requested 2024-11-24T02:47:01,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421547 2024-11-24T02:47:01,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,552 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,552 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421547 2024-11-24T02:47:01,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,553 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,553 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421547) roll requested 2024-11-24T02:47:01,553 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421553 2024-11-24T02:47:01,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421553 2024-11-24T02:47:01,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,557 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421553) roll requested 2024-11-24T02:47:01,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421558 2024-11-24T02:47:01,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421553 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421558 2024-11-24T02:47:01,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,567 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421558) roll requested 2024-11-24T02:47:01,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421567 2024-11-24T02:47:01,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421558 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421567 2024-11-24T02:47:01,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,576 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,577 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,577 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421567) roll requested 2024-11-24T02:47:01,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421577 2024-11-24T02:47:01,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421577 2024-11-24T02:47:01,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421577) roll requested 2024-11-24T02:47:01,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421583 2024-11-24T02:47:01,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421577 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421583 2024-11-24T02:47:01,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,591 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421583) roll requested 2024-11-24T02:47:01,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421591 2024-11-24T02:47:01,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421591 2024-11-24T02:47:01,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,601 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421591) roll requested 2024-11-24T02:47:01,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421602 2024-11-24T02:47:01,607 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,607 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,607 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421591 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421602 2024-11-24T02:47:01,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,607 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421602) roll requested 2024-11-24T02:47:01,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421608 2024-11-24T02:47:01,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421602 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421608 2024-11-24T02:47:01,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421608) roll requested 2024-11-24T02:47:01,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421613 2024-11-24T02:47:01,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421608 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421613 2024-11-24T02:47:01,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421613) roll requested 2024-11-24T02:47:01,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421619 2024-11-24T02:47:01,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421613 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421619 2024-11-24T02:47:01,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,625 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421619) roll requested 2024-11-24T02:47:01,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421625 2024-11-24T02:47:01,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,632 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421625 2024-11-24T02:47:01,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421625) roll requested 2024-11-24T02:47:01,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421633 2024-11-24T02:47:01,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421625 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421633 2024-11-24T02:47:01,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,639 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421633) roll requested 2024-11-24T02:47:01,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421640 2024-11-24T02:47:01,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421640 2024-11-24T02:47:01,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,652 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421640) roll requested 2024-11-24T02:47:01,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421653 2024-11-24T02:47:01,675 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,675 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,675 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421640 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421653 2024-11-24T02:47:01,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,677 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,677 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421653) roll requested 2024-11-24T02:47:01,677 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421677 2024-11-24T02:47:01,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421677 2024-11-24T02:47:01,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421677) roll requested 2024-11-24T02:47:01,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421703 2024-11-24T02:47:01,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,711 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,711 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421677 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421703 2024-11-24T02:47:01,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421703) roll requested 2024-11-24T02:47:01,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421713 2024-11-24T02:47:01,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421713 2024-11-24T02:47:01,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421713) roll requested 2024-11-24T02:47:01,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421724 2024-11-24T02:47:01,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421724 2024-11-24T02:47:01,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,733 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,733 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421724) roll requested 2024-11-24T02:47:01,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421734 2024-11-24T02:47:01,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421734 2024-11-24T02:47:01,745 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,745 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,745 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421734) roll requested 2024-11-24T02:47:01,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421745 2024-11-24T02:47:01,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421734 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421745 2024-11-24T02:47:01,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421745) roll requested 2024-11-24T02:47:01,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421760 2024-11-24T02:47:01,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421760 2024-11-24T02:47:01,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421760) roll requested 2024-11-24T02:47:01,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421772 2024-11-24T02:47:01,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421772 2024-11-24T02:47:01,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,779 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421772) roll requested 2024-11-24T02:47:01,780 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421779 2024-11-24T02:47:01,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421779 2024-11-24T02:47:01,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,800 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421779) roll requested 2024-11-24T02:47:01,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421800 2024-11-24T02:47:01,805 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,805 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421800 2024-11-24T02:47:01,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,806 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421800) roll requested 2024-11-24T02:47:01,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421806 2024-11-24T02:47:01,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421800 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421806 2024-11-24T02:47:01,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,812 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421806) roll requested 2024-11-24T02:47:01,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421812 2024-11-24T02:47:01,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421806 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421812 2024-11-24T02:47:01,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421812) roll requested 2024-11-24T02:47:01,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421820 2024-11-24T02:47:01,826 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,826 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,826 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,826 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,826 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,826 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421812 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421820 2024-11-24T02:47:01,827 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,827 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,827 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,827 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421820) roll requested 2024-11-24T02:47:01,827 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421827 2024-11-24T02:47:01,832 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,832 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421827 2024-11-24T02:47:01,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,835 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421827) roll requested 2024-11-24T02:47:01,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421836 2024-11-24T02:47:01,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,844 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,844 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421827 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421836 2024-11-24T02:47:01,845 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,845 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,845 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421836) roll requested 2024-11-24T02:47:01,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421845 2024-11-24T02:47:01,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421845 2024-11-24T02:47:01,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,857 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421845) roll requested 2024-11-24T02:47:01,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421858 2024-11-24T02:47:01,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421845 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421858 2024-11-24T02:47:01,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,864 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421858) roll requested 2024-11-24T02:47:01,864 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421864 2024-11-24T02:47:01,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421858 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421864 2024-11-24T02:47:01,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,869 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421864) roll requested 2024-11-24T02:47:01,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421869 2024-11-24T02:47:01,876 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,876 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,876 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421864 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421869 2024-11-24T02:47:01,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,879 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,879 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421869) roll requested 2024-11-24T02:47:01,879 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421879 2024-11-24T02:47:01,884 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,884 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,884 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,884 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,884 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,884 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421869 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421879 2024-11-24T02:47:01,885 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,885 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,885 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,885 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421879) roll requested 2024-11-24T02:47:01,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421885 2024-11-24T02:47:01,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,891 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,891 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421879 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421885 2024-11-24T02:47:01,893 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,893 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421885) roll requested 2024-11-24T02:47:01,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421893 2024-11-24T02:47:01,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,901 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,901 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421885 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421893 2024-11-24T02:47:01,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,903 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421893) roll requested 2024-11-24T02:47:01,903 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421903 2024-11-24T02:47:01,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421893 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421903 2024-11-24T02:47:01,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421903) roll requested 2024-11-24T02:47:01,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421924 2024-11-24T02:47:01,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,943 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,943 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421903 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421924 2024-11-24T02:47:01,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421924) roll requested 2024-11-24T02:47:01,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421945 2024-11-24T02:47:01,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421945 2024-11-24T02:47:01,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421945) roll requested 2024-11-24T02:47:01,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421960 2024-11-24T02:47:01,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421960 2024-11-24T02:47:01,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,967 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421960) roll requested 2024-11-24T02:47:01,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421967 2024-11-24T02:47:01,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421967 2024-11-24T02:47:01,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,978 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421967) roll requested 2024-11-24T02:47:01,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421978 2024-11-24T02:47:01,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421967 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421978 2024-11-24T02:47:01,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421978) roll requested 2024-11-24T02:47:01,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421989 2024-11-24T02:47:01,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:01,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421978 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421989 2024-11-24T02:47:01,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:01,996 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,996 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:01,996 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421989) roll requested 2024-11-24T02:47:01,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416421996 2024-11-24T02:47:02,002 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,002 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,002 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421996 2024-11-24T02:47:02,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416421996) roll requested 2024-11-24T02:47:02,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422004 2024-11-24T02:47:02,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416421996 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422004 2024-11-24T02:47:02,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422004) roll requested 2024-11-24T02:47:02,022 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422021 2024-11-24T02:47:02,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422021 2024-11-24T02:47:02,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422021) roll requested 2024-11-24T02:47:02,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422030 2024-11-24T02:47:02,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422030 2024-11-24T02:47:02,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422030) roll requested 2024-11-24T02:47:02,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422050 2024-11-24T02:47:02,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422050 2024-11-24T02:47:02,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,068 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422050) roll requested 2024-11-24T02:47:02,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422068 2024-11-24T02:47:02,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422050 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422068 2024-11-24T02:47:02,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422068) roll requested 2024-11-24T02:47:02,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422080 2024-11-24T02:47:02,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422068 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422080 2024-11-24T02:47:02,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422080) roll requested 2024-11-24T02:47:02,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422089 2024-11-24T02:47:02,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422089 2024-11-24T02:47:02,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422089) roll requested 2024-11-24T02:47:02,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422106 2024-11-24T02:47:02,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422106 2024-11-24T02:47:02,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,116 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422106) roll requested 2024-11-24T02:47:02,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422116 2024-11-24T02:47:02,124 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,124 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,124 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422116 2024-11-24T02:47:02,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422116) roll requested 2024-11-24T02:47:02,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422129 2024-11-24T02:47:02,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422129 2024-11-24T02:47:02,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422129) roll requested 2024-11-24T02:47:02,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422141 2024-11-24T02:47:02,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422141 2024-11-24T02:47:02,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,149 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422141) roll requested 2024-11-24T02:47:02,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422149 2024-11-24T02:47:02,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422149 2024-11-24T02:47:02,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422149) roll requested 2024-11-24T02:47:02,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422160 2024-11-24T02:47:02,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422160 2024-11-24T02:47:02,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,168 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422160) roll requested 2024-11-24T02:47:02,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422168 2024-11-24T02:47:02,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422168 2024-11-24T02:47:02,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422168) roll requested 2024-11-24T02:47:02,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422180 2024-11-24T02:47:02,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422168 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422180 2024-11-24T02:47:02,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422180) roll requested 2024-11-24T02:47:02,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422187 2024-11-24T02:47:02,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422187 2024-11-24T02:47:02,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422187) roll requested 2024-11-24T02:47:02,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422197 2024-11-24T02:47:02,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,206 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,206 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422187 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422197 2024-11-24T02:47:02,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422197) roll requested 2024-11-24T02:47:02,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422208 2024-11-24T02:47:02,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422208 2024-11-24T02:47:02,215 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422208) roll requested 2024-11-24T02:47:02,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422216 2024-11-24T02:47:02,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422216 2024-11-24T02:47:02,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,221 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422216) roll requested 2024-11-24T02:47:02,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422222 2024-11-24T02:47:02,233 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,233 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,233 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,233 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,233 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,233 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422222 2024-11-24T02:47:02,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422222) roll requested 2024-11-24T02:47:02,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422236 2024-11-24T02:47:02,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422222 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422236 2024-11-24T02:47:02,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,244 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422236) roll requested 2024-11-24T02:47:02,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422245 2024-11-24T02:47:02,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422245 2024-11-24T02:47:02,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422245) roll requested 2024-11-24T02:47:02,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422253 2024-11-24T02:47:02,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,260 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,260 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422253 2024-11-24T02:47:02,261 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,261 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,261 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422253) roll requested 2024-11-24T02:47:02,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422261 2024-11-24T02:47:02,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422261 2024-11-24T02:47:02,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,268 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422261) roll requested 2024-11-24T02:47:02,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422269 2024-11-24T02:47:02,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,274 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,274 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,274 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422261 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422269 2024-11-24T02:47:02,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,276 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422269) roll requested 2024-11-24T02:47:02,276 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422276 2024-11-24T02:47:02,281 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,281 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,281 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,281 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,281 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,281 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422269 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422276 2024-11-24T02:47:02,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,284 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422276) roll requested 2024-11-24T02:47:02,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422284 2024-11-24T02:47:02,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422276 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422284 2024-11-24T02:47:02,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422284) roll requested 2024-11-24T02:47:02,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422291 2024-11-24T02:47:02,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422291 2024-11-24T02:47:02,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422291) roll requested 2024-11-24T02:47:02,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422297 2024-11-24T02:47:02,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422297 2024-11-24T02:47:02,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,304 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422297) roll requested 2024-11-24T02:47:02,304 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422304 2024-11-24T02:47:02,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,310 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,310 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,310 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,310 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422304 2024-11-24T02:47:02,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,312 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422304) roll requested 2024-11-24T02:47:02,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422312 2024-11-24T02:47:02,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422304 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422312 2024-11-24T02:47:02,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422312) roll requested 2024-11-24T02:47:02,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422378 2024-11-24T02:47:02,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422312 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422378 2024-11-24T02:47:02,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422378) roll requested 2024-11-24T02:47:02,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422392 2024-11-24T02:47:02,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,398 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422392 2024-11-24T02:47:02,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,399 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,399 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,399 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422392) roll requested 2024-11-24T02:47:02,399 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422399 2024-11-24T02:47:02,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422392 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422399 2024-11-24T02:47:02,409 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,409 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,409 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,409 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422399) roll requested 2024-11-24T02:47:02,409 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422409 2024-11-24T02:47:02,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422399 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422409 2024-11-24T02:47:02,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422409) roll requested 2024-11-24T02:47:02,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422417 2024-11-24T02:47:02,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422409 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422417 2024-11-24T02:47:02,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422417) roll requested 2024-11-24T02:47:02,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422426 2024-11-24T02:47:02,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,436 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,436 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,436 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422417 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422426 2024-11-24T02:47:02,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,436 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,437 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422426) roll requested 2024-11-24T02:47:02,437 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422437 2024-11-24T02:47:02,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,441 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422437 2024-11-24T02:47:02,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422437) roll requested 2024-11-24T02:47:02,442 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422442 2024-11-24T02:47:02,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422437 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422442 2024-11-24T02:47:02,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422442) roll requested 2024-11-24T02:47:02,449 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422449 2024-11-24T02:47:02,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422442 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422449 2024-11-24T02:47:02,456 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,456 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422449) roll requested 2024-11-24T02:47:02,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422456 2024-11-24T02:47:02,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,461 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,461 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,462 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422449 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422456 2024-11-24T02:47:02,462 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,462 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,462 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422456) roll requested 2024-11-24T02:47:02,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422462 2024-11-24T02:47:02,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422462 2024-11-24T02:47:02,473 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,473 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422462) roll requested 2024-11-24T02:47:02,474 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422474 2024-11-24T02:47:02,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,479 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422462 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422474 2024-11-24T02:47:02,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,480 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422474) roll requested 2024-11-24T02:47:02,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422480 2024-11-24T02:47:02,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422474 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422480 2024-11-24T02:47:02,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422480) roll requested 2024-11-24T02:47:02,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422486 2024-11-24T02:47:02,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422480 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422486 2024-11-24T02:47:02,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422486) roll requested 2024-11-24T02:47:02,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422492 2024-11-24T02:47:02,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,498 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,498 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422486 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422492 2024-11-24T02:47:02,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422492) roll requested 2024-11-24T02:47:02,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422499 2024-11-24T02:47:02,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,506 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422499 2024-11-24T02:47:02,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,508 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422499) roll requested 2024-11-24T02:47:02,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422508 2024-11-24T02:47:02,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422508 2024-11-24T02:47:02,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,517 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422508) roll requested 2024-11-24T02:47:02,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422517 2024-11-24T02:47:02,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422517 2024-11-24T02:47:02,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422517) roll requested 2024-11-24T02:47:02,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422523 2024-11-24T02:47:02,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422517 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422523 2024-11-24T02:47:02,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422523) roll requested 2024-11-24T02:47:02,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422531 2024-11-24T02:47:02,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422531 2024-11-24T02:47:02,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422531) roll requested 2024-11-24T02:47:02,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422541 2024-11-24T02:47:02,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422541 2024-11-24T02:47:02,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422541) roll requested 2024-11-24T02:47:02,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422548 2024-11-24T02:47:02,553 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,553 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422548 2024-11-24T02:47:02,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,554 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422548) roll requested 2024-11-24T02:47:02,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422554 2024-11-24T02:47:02,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422548 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422554 2024-11-24T02:47:02,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422554) roll requested 2024-11-24T02:47:02,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422560 2024-11-24T02:47:02,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,565 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,565 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422554 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422560 2024-11-24T02:47:02,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422560) roll requested 2024-11-24T02:47:02,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422567 2024-11-24T02:47:02,572 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,572 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,572 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,572 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422567 2024-11-24T02:47:02,572 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,572 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422567) roll requested 2024-11-24T02:47:02,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422573 2024-11-24T02:47:02,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,578 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,578 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,578 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,578 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422573 2024-11-24T02:47:02,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422573) roll requested 2024-11-24T02:47:02,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422580 2024-11-24T02:47:02,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422573 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422580 2024-11-24T02:47:02,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,588 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,588 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422580) roll requested 2024-11-24T02:47:02,588 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422588 2024-11-24T02:47:02,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422588 2024-11-24T02:47:02,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422588) roll requested 2024-11-24T02:47:02,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422598 2024-11-24T02:47:02,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422588 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422598 2024-11-24T02:47:02,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422598) roll requested 2024-11-24T02:47:02,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422606 2024-11-24T02:47:02,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422606 2024-11-24T02:47:02,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422606) roll requested 2024-11-24T02:47:02,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422616 2024-11-24T02:47:02,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,621 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,621 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,621 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,621 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422616 2024-11-24T02:47:02,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422616) roll requested 2024-11-24T02:47:02,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422623 2024-11-24T02:47:02,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422623 2024-11-24T02:47:02,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422623) roll requested 2024-11-24T02:47:02,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422629 2024-11-24T02:47:02,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422629 2024-11-24T02:47:02,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,636 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422629) roll requested 2024-11-24T02:47:02,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422636 2024-11-24T02:47:02,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422636 2024-11-24T02:47:02,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,643 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422636) roll requested 2024-11-24T02:47:02,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422643 2024-11-24T02:47:02,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422636 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422643 2024-11-24T02:47:02,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422643) roll requested 2024-11-24T02:47:02,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422653 2024-11-24T02:47:02,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,659 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,659 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,659 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,659 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,659 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422643 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422653 2024-11-24T02:47:02,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,660 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,660 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422653) roll requested 2024-11-24T02:47:02,660 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422660 2024-11-24T02:47:02,664 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,664 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,665 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,665 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422653 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422660 2024-11-24T02:47:02,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,669 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422660) roll requested 2024-11-24T02:47:02,669 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422669 2024-11-24T02:47:02,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422660 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422669 2024-11-24T02:47:02,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422669) roll requested 2024-11-24T02:47:02,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422674 2024-11-24T02:47:02,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422669 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422674 2024-11-24T02:47:02,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,684 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422674) roll requested 2024-11-24T02:47:02,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422684 2024-11-24T02:47:02,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422684 2024-11-24T02:47:02,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,690 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422684) roll requested 2024-11-24T02:47:02,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422690 2024-11-24T02:47:02,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422684 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422690 2024-11-24T02:47:02,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,698 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,698 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,698 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422690) roll requested 2024-11-24T02:47:02,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422698 2024-11-24T02:47:02,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422698 2024-11-24T02:47:02,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,704 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422698) roll requested 2024-11-24T02:47:02,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422705 2024-11-24T02:47:02,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422698 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422705 2024-11-24T02:47:02,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,712 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422705) roll requested 2024-11-24T02:47:02,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422713 2024-11-24T02:47:02,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,718 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422705 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422713 2024-11-24T02:47:02,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,719 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422713) roll requested 2024-11-24T02:47:02,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422719 2024-11-24T02:47:02,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422719 2024-11-24T02:47:02,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422719) roll requested 2024-11-24T02:47:02,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422724 2024-11-24T02:47:02,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422724 2024-11-24T02:47:02,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,740 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,740 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422724) roll requested 2024-11-24T02:47:02,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422740 2024-11-24T02:47:02,747 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,747 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,747 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,747 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422740 2024-11-24T02:47:02,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,748 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,748 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422740) roll requested 2024-11-24T02:47:02,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422748 2024-11-24T02:47:02,754 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,754 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,754 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,754 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,754 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,754 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422740 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422748 2024-11-24T02:47:02,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,756 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,756 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422748) roll requested 2024-11-24T02:47:02,756 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422756 2024-11-24T02:47:02,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422748 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422756 2024-11-24T02:47:02,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,763 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,763 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422756) roll requested 2024-11-24T02:47:02,763 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422763 2024-11-24T02:47:02,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422756 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422763 2024-11-24T02:47:02,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,769 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422763) roll requested 2024-11-24T02:47:02,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422769 2024-11-24T02:47:02,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422763 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422769 2024-11-24T02:47:02,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,776 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422769) roll requested 2024-11-24T02:47:02,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422777 2024-11-24T02:47:02,782 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,782 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,782 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,782 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,782 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,782 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422769 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422777 2024-11-24T02:47:02,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422777) roll requested 2024-11-24T02:47:02,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422784 2024-11-24T02:47:02,796 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422777 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422784 2024-11-24T02:47:02,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,808 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422784) roll requested 2024-11-24T02:47:02,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422809 2024-11-24T02:47:02,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,825 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,825 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,825 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422809 2024-11-24T02:47:02,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,825 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,826 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422809) roll requested 2024-11-24T02:47:02,826 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422826 2024-11-24T02:47:02,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422826 2024-11-24T02:47:02,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422826) roll requested 2024-11-24T02:47:02,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422862 2024-11-24T02:47:02,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422826 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422862 2024-11-24T02:47:02,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422862) roll requested 2024-11-24T02:47:02,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422909 2024-11-24T02:47:02,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422909 2024-11-24T02:47:02,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,948 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422909) roll requested 2024-11-24T02:47:02,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422949 2024-11-24T02:47:02,974 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422909 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422949 2024-11-24T02:47:02,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:02,985 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:02,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422949) roll requested 2024-11-24T02:47:02,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416422985 2024-11-24T02:47:02,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:02,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422949 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422985 2024-11-24T02:47:03,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416422985) roll requested 2024-11-24T02:47:03,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423000 2024-11-24T02:47:03,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416422985 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423000 2024-11-24T02:47:03,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423000) roll requested 2024-11-24T02:47:03,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423007 2024-11-24T02:47:03,015 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423000 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423007 2024-11-24T02:47:03,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,017 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423007) roll requested 2024-11-24T02:47:03,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423017 2024-11-24T02:47:03,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423017 2024-11-24T02:47:03,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423017) roll requested 2024-11-24T02:47:03,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423028 2024-11-24T02:47:03,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423017 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423028 2024-11-24T02:47:03,043 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,043 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423028) roll requested 2024-11-24T02:47:03,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423044 2024-11-24T02:47:03,052 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,052 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,052 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,052 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423044 2024-11-24T02:47:03,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423044) roll requested 2024-11-24T02:47:03,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423057 2024-11-24T02:47:03,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423044 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423057 2024-11-24T02:47:03,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,069 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423057) roll requested 2024-11-24T02:47:03,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423070 2024-11-24T02:47:03,076 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,076 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,076 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,076 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,076 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,076 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423057 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423070 2024-11-24T02:47:03,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,076 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,077 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,077 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423070) roll requested 2024-11-24T02:47:03,077 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423077 2024-11-24T02:47:03,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,084 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423077 2024-11-24T02:47:03,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423077) roll requested 2024-11-24T02:47:03,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423089 2024-11-24T02:47:03,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,101 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,101 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423077 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423089 2024-11-24T02:47:03,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423089) roll requested 2024-11-24T02:47:03,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423102 2024-11-24T02:47:03,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423089 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423102 2024-11-24T02:47:03,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423102) roll requested 2024-11-24T02:47:03,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423113 2024-11-24T02:47:03,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423113 2024-11-24T02:47:03,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423113) roll requested 2024-11-24T02:47:03,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423122 2024-11-24T02:47:03,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423122 2024-11-24T02:47:03,132 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,132 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,133 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423122) roll requested 2024-11-24T02:47:03,133 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423133 2024-11-24T02:47:03,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,151 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,151 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,151 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423122 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423133 2024-11-24T02:47:03,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,160 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423133) roll requested 2024-11-24T02:47:03,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423160 2024-11-24T02:47:03,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423133 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423160 2024-11-24T02:47:03,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,180 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423160) roll requested 2024-11-24T02:47:03,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423180 2024-11-24T02:47:03,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423180 2024-11-24T02:47:03,189 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,189 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,189 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,189 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423180) roll requested 2024-11-24T02:47:03,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423189 2024-11-24T02:47:03,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423189 2024-11-24T02:47:03,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,200 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423189) roll requested 2024-11-24T02:47:03,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423200 2024-11-24T02:47:03,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423189 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423200 2024-11-24T02:47:03,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,205 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423200) roll requested 2024-11-24T02:47:03,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423205 2024-11-24T02:47:03,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423200 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423205 2024-11-24T02:47:03,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,216 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423205) roll requested 2024-11-24T02:47:03,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423216 2024-11-24T02:47:03,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,221 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423205 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423216 2024-11-24T02:47:03,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423216) roll requested 2024-11-24T02:47:03,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423226 2024-11-24T02:47:03,231 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,232 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423216 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423226 2024-11-24T02:47:03,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,232 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423226) roll requested 2024-11-24T02:47:03,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423232 2024-11-24T02:47:03,238 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423232 2024-11-24T02:47:03,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,239 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423232) roll requested 2024-11-24T02:47:03,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423239 2024-11-24T02:47:03,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423232 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423239 2024-11-24T02:47:03,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,252 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423239) roll requested 2024-11-24T02:47:03,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423252 2024-11-24T02:47:03,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423239 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423252 2024-11-24T02:47:03,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,264 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423252) roll requested 2024-11-24T02:47:03,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423264 2024-11-24T02:47:03,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423252 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423264 2024-11-24T02:47:03,270 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,270 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,270 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,270 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423264) roll requested 2024-11-24T02:47:03,270 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423270 2024-11-24T02:47:03,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423264 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423270 2024-11-24T02:47:03,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423270) roll requested 2024-11-24T02:47:03,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423275 2024-11-24T02:47:03,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423270 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423275 2024-11-24T02:47:03,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423275) roll requested 2024-11-24T02:47:03,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423281 2024-11-24T02:47:03,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423281 2024-11-24T02:47:03,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,288 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423281) roll requested 2024-11-24T02:47:03,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423288 2024-11-24T02:47:03,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,297 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,297 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,297 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423288 2024-11-24T02:47:03,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,297 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423288) roll requested 2024-11-24T02:47:03,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423297 2024-11-24T02:47:03,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423288 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423297 2024-11-24T02:47:03,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423297) roll requested 2024-11-24T02:47:03,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423305 2024-11-24T02:47:03,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423305 2024-11-24T02:47:03,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,315 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423305) roll requested 2024-11-24T02:47:03,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423315 2024-11-24T02:47:03,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423315 2024-11-24T02:47:03,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423315) roll requested 2024-11-24T02:47:03,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423323 2024-11-24T02:47:03,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423315 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423323 2024-11-24T02:47:03,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423323) roll requested 2024-11-24T02:47:03,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423334 2024-11-24T02:47:03,340 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423334 2024-11-24T02:47:03,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423334) roll requested 2024-11-24T02:47:03,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423342 2024-11-24T02:47:03,352 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423342 2024-11-24T02:47:03,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423342) roll requested 2024-11-24T02:47:03,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423356 2024-11-24T02:47:03,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423356 2024-11-24T02:47:03,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423356) roll requested 2024-11-24T02:47:03,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423371 2024-11-24T02:47:03,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423371 2024-11-24T02:47:03,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,379 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423371) roll requested 2024-11-24T02:47:03,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423379 2024-11-24T02:47:03,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423379 2024-11-24T02:47:03,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,417 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423379) roll requested 2024-11-24T02:47:03,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423419 2024-11-24T02:47:03,434 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423419 2024-11-24T02:47:03,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,444 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423419) roll requested 2024-11-24T02:47:03,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423445 2024-11-24T02:47:03,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423445 2024-11-24T02:47:03,495 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,495 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,495 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423445) roll requested 2024-11-24T02:47:03,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423499 2024-11-24T02:47:03,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423499 2024-11-24T02:47:03,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423499) roll requested 2024-11-24T02:47:03,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423521 2024-11-24T02:47:03,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423499 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423521 2024-11-24T02:47:03,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,532 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423521) roll requested 2024-11-24T02:47:03,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423532 2024-11-24T02:47:03,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423521 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423532 2024-11-24T02:47:03,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,544 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423532) roll requested 2024-11-24T02:47:03,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423544 2024-11-24T02:47:03,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,552 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,552 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423532 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423544 2024-11-24T02:47:03,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423544) roll requested 2024-11-24T02:47:03,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423555 2024-11-24T02:47:03,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423544 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423555 2024-11-24T02:47:03,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423555) roll requested 2024-11-24T02:47:03,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423563 2024-11-24T02:47:03,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423563 2024-11-24T02:47:03,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423563) roll requested 2024-11-24T02:47:03,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423574 2024-11-24T02:47:03,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423574 2024-11-24T02:47:03,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,581 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423574) roll requested 2024-11-24T02:47:03,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423581 2024-11-24T02:47:03,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423581 2024-11-24T02:47:03,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423581) roll requested 2024-11-24T02:47:03,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423589 2024-11-24T02:47:03,596 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423589 2024-11-24T02:47:03,597 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,597 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423589) roll requested 2024-11-24T02:47:03,598 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423598 2024-11-24T02:47:03,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423598 2024-11-24T02:47:03,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423598) roll requested 2024-11-24T02:47:03,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423610 2024-11-24T02:47:03,615 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423598 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423610 2024-11-24T02:47:03,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423610) roll requested 2024-11-24T02:47:03,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423617 2024-11-24T02:47:03,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423617 2024-11-24T02:47:03,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,626 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423617) roll requested 2024-11-24T02:47:03,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423627 2024-11-24T02:47:03,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423617 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423627 2024-11-24T02:47:03,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423627) roll requested 2024-11-24T02:47:03,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423644 2024-11-24T02:47:03,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423627 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423644 2024-11-24T02:47:03,659 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,659 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423644) roll requested 2024-11-24T02:47:03,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423659 2024-11-24T02:47:03,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,667 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423659 2024-11-24T02:47:03,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423659) roll requested 2024-11-24T02:47:03,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423668 2024-11-24T02:47:03,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423659 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423668 2024-11-24T02:47:03,676 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,676 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,676 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423668) roll requested 2024-11-24T02:47:03,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423676 2024-11-24T02:47:03,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,682 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,682 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423676 2024-11-24T02:47:03,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,683 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,683 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423676) roll requested 2024-11-24T02:47:03,683 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423683 2024-11-24T02:47:03,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423676 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423683 2024-11-24T02:47:03,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423683) roll requested 2024-11-24T02:47:03,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423689 2024-11-24T02:47:03,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,697 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,697 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423683 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423689 2024-11-24T02:47:03,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423689) roll requested 2024-11-24T02:47:03,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423700 2024-11-24T02:47:03,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,706 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,706 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423700 2024-11-24T02:47:03,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,708 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423700) roll requested 2024-11-24T02:47:03,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423708 2024-11-24T02:47:03,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423708 2024-11-24T02:47:03,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423708) roll requested 2024-11-24T02:47:03,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423714 2024-11-24T02:47:03,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423714 2024-11-24T02:47:03,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423714) roll requested 2024-11-24T02:47:03,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423724 2024-11-24T02:47:03,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423714 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423724 2024-11-24T02:47:03,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,732 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423724) roll requested 2024-11-24T02:47:03,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423732 2024-11-24T02:47:03,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,738 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,738 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,738 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423724 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423732 2024-11-24T02:47:03,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,738 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423732) roll requested 2024-11-24T02:47:03,739 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423739 2024-11-24T02:47:03,744 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,744 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,744 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,745 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,745 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423732 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423739 2024-11-24T02:47:03,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,747 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,747 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423739) roll requested 2024-11-24T02:47:03,747 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423747 2024-11-24T02:47:03,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423739 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423747 2024-11-24T02:47:03,754 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,754 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,754 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,755 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423747) roll requested 2024-11-24T02:47:03,755 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423755 2024-11-24T02:47:03,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423747 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423755 2024-11-24T02:47:03,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,760 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423755) roll requested 2024-11-24T02:47:03,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423761 2024-11-24T02:47:03,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423755 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423761 2024-11-24T02:47:03,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423761) roll requested 2024-11-24T02:47:03,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423772 2024-11-24T02:47:03,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423761 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423772 2024-11-24T02:47:03,783 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,783 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,783 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,783 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423772) roll requested 2024-11-24T02:47:03,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423783 2024-11-24T02:47:03,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423772 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423783 2024-11-24T02:47:03,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,792 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423783) roll requested 2024-11-24T02:47:03,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423793 2024-11-24T02:47:03,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,801 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,801 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,801 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423783 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423793 2024-11-24T02:47:03,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423793) roll requested 2024-11-24T02:47:03,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423802 2024-11-24T02:47:03,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423802 2024-11-24T02:47:03,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423802) roll requested 2024-11-24T02:47:03,814 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423814 2024-11-24T02:47:03,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423814 2024-11-24T02:47:03,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423814) roll requested 2024-11-24T02:47:03,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423824 2024-11-24T02:47:03,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423814 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423824 2024-11-24T02:47:03,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423824) roll requested 2024-11-24T02:47:03,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423829 2024-11-24T02:47:03,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423829 2024-11-24T02:47:03,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,834 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,834 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423829) roll requested 2024-11-24T02:47:03,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423835 2024-11-24T02:47:03,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423835 2024-11-24T02:47:03,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423835) roll requested 2024-11-24T02:47:03,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423842 2024-11-24T02:47:03,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423835 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423842 2024-11-24T02:47:03,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423842) roll requested 2024-11-24T02:47:03,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423849 2024-11-24T02:47:03,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423849 2024-11-24T02:47:03,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,854 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423849) roll requested 2024-11-24T02:47:03,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423855 2024-11-24T02:47:03,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,860 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423855 2024-11-24T02:47:03,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,861 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423855) roll requested 2024-11-24T02:47:03,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423861 2024-11-24T02:47:03,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423861 2024-11-24T02:47:03,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423861) roll requested 2024-11-24T02:47:03,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423868 2024-11-24T02:47:03,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423861 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423868 2024-11-24T02:47:03,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,875 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423868) roll requested 2024-11-24T02:47:03,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423875 2024-11-24T02:47:03,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423875 2024-11-24T02:47:03,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423875) roll requested 2024-11-24T02:47:03,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423883 2024-11-24T02:47:03,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423875 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423883 2024-11-24T02:47:03,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,892 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423883) roll requested 2024-11-24T02:47:03,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423892 2024-11-24T02:47:03,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423892 2024-11-24T02:47:03,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,901 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423892) roll requested 2024-11-24T02:47:03,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423901 2024-11-24T02:47:03,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423892 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423901 2024-11-24T02:47:03,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423901) roll requested 2024-11-24T02:47:03,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423911 2024-11-24T02:47:03,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,917 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,917 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,917 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423901 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423911 2024-11-24T02:47:03,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,921 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423911) roll requested 2024-11-24T02:47:03,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423921 2024-11-24T02:47:03,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423921 2024-11-24T02:47:03,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,927 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423921) roll requested 2024-11-24T02:47:03,927 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423927 2024-11-24T02:47:03,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423921 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423927 2024-11-24T02:47:03,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423927) roll requested 2024-11-24T02:47:03,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423939 2024-11-24T02:47:03,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,944 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423927 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423939 2024-11-24T02:47:03,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423939) roll requested 2024-11-24T02:47:03,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423947 2024-11-24T02:47:03,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423947 2024-11-24T02:47:03,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,953 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423947) roll requested 2024-11-24T02:47:03,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423953 2024-11-24T02:47:03,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423953 2024-11-24T02:47:03,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,960 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423953) roll requested 2024-11-24T02:47:03,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423960 2024-11-24T02:47:03,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423960 2024-11-24T02:47:03,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,966 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423960) roll requested 2024-11-24T02:47:03,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423966 2024-11-24T02:47:03,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423966 2024-11-24T02:47:03,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423966) roll requested 2024-11-24T02:47:03,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423976 2024-11-24T02:47:03,980 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,980 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,980 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423966 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423976 2024-11-24T02:47:03,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,981 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,981 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423976) roll requested 2024-11-24T02:47:03,981 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423981 2024-11-24T02:47:03,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,986 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,986 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423981 2024-11-24T02:47:03,986 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,986 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423981) roll requested 2024-11-24T02:47:03,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423987 2024-11-24T02:47:03,991 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,991 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423981 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423987 2024-11-24T02:47:03,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,992 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,992 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423987) roll requested 2024-11-24T02:47:03,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423992 2024-11-24T02:47:03,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:03,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423987 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423992 2024-11-24T02:47:03,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:03,997 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:03,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423992) roll requested 2024-11-24T02:47:03,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416423997 2024-11-24T02:47:04,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423992 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423997 2024-11-24T02:47:04,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,002 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416423997) roll requested 2024-11-24T02:47:04,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424002 2024-11-24T02:47:04,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416423997 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424002 2024-11-24T02:47:04,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,007 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424002) roll requested 2024-11-24T02:47:04,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424007 2024-11-24T02:47:04,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424002 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424007 2024-11-24T02:47:04,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424007) roll requested 2024-11-24T02:47:04,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424013 2024-11-24T02:47:04,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424007 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424013 2024-11-24T02:47:04,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,020 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424013) roll requested 2024-11-24T02:47:04,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424020 2024-11-24T02:47:04,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424013 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424020 2024-11-24T02:47:04,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424020) roll requested 2024-11-24T02:47:04,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424028 2024-11-24T02:47:04,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424028 2024-11-24T02:47:04,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,034 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424028) roll requested 2024-11-24T02:47:04,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424034 2024-11-24T02:47:04,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424028 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424034 2024-11-24T02:47:04,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,039 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,039 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424034) roll requested 2024-11-24T02:47:04,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424039 2024-11-24T02:47:04,044 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,044 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424034 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424039 2024-11-24T02:47:04,045 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,045 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424039) roll requested 2024-11-24T02:47:04,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424045 2024-11-24T02:47:04,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424039 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424045 2024-11-24T02:47:04,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,050 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424045) roll requested 2024-11-24T02:47:04,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424051 2024-11-24T02:47:04,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424045 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424051 2024-11-24T02:47:04,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,056 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424051) roll requested 2024-11-24T02:47:04,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424056 2024-11-24T02:47:04,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424056 2024-11-24T02:47:04,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,061 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424056) roll requested 2024-11-24T02:47:04,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424061 2024-11-24T02:47:04,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424056 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424061 2024-11-24T02:47:04,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424061) roll requested 2024-11-24T02:47:04,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424067 2024-11-24T02:47:04,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424061 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424067 2024-11-24T02:47:04,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,072 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424067) roll requested 2024-11-24T02:47:04,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424073 2024-11-24T02:47:04,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424067 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424073 2024-11-24T02:47:04,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424073) roll requested 2024-11-24T02:47:04,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424078 2024-11-24T02:47:04,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,082 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424078 2024-11-24T02:47:04,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,083 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,083 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,083 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424078) roll requested 2024-11-24T02:47:04,083 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424083 2024-11-24T02:47:04,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,088 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,088 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424083 2024-11-24T02:47:04,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,088 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424083) roll requested 2024-11-24T02:47:04,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424088 2024-11-24T02:47:04,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424083 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424088 2024-11-24T02:47:04,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,094 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424088) roll requested 2024-11-24T02:47:04,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424094 2024-11-24T02:47:04,099 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,099 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,099 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,099 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,099 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,099 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424094 2024-11-24T02:47:04,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,099 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,100 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,102 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424094) roll requested 2024-11-24T02:47:04,102 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424102 2024-11-24T02:47:04,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424094 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424102 2024-11-24T02:47:04,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,108 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,108 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424102) roll requested 2024-11-24T02:47:04,108 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424108 2024-11-24T02:47:04,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,112 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,112 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,112 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,112 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,112 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424102 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424108 2024-11-24T02:47:04,113 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,113 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,113 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,113 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424108) roll requested 2024-11-24T02:47:04,113 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424113 2024-11-24T02:47:04,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,118 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,118 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,118 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424108 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424113 2024-11-24T02:47:04,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424113) roll requested 2024-11-24T02:47:04,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424119 2024-11-24T02:47:04,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424113 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424119 2024-11-24T02:47:04,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,124 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424119) roll requested 2024-11-24T02:47:04,124 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424124 2024-11-24T02:47:04,129 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,129 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,129 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,129 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,129 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,129 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424119 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424124 2024-11-24T02:47:04,130 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,130 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,130 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424124) roll requested 2024-11-24T02:47:04,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424130 2024-11-24T02:47:04,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424124 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424130 2024-11-24T02:47:04,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,136 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,137 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424130) roll requested 2024-11-24T02:47:04,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424137 2024-11-24T02:47:04,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424130 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424137 2024-11-24T02:47:04,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,143 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424137) roll requested 2024-11-24T02:47:04,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424143 2024-11-24T02:47:04,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424137 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424143 2024-11-24T02:47:04,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424143) roll requested 2024-11-24T02:47:04,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424148 2024-11-24T02:47:04,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424143 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424148 2024-11-24T02:47:04,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,154 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424148) roll requested 2024-11-24T02:47:04,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424154 2024-11-24T02:47:04,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,159 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,159 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,159 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,159 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424154 2024-11-24T02:47:04,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424154) roll requested 2024-11-24T02:47:04,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424160 2024-11-24T02:47:04,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424154 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424160 2024-11-24T02:47:04,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,165 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424160) roll requested 2024-11-24T02:47:04,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424165 2024-11-24T02:47:04,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424160 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424165 2024-11-24T02:47:04,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424165) roll requested 2024-11-24T02:47:04,170 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424170 2024-11-24T02:47:04,174 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,174 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,174 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,174 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,174 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,174 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424165 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424170 2024-11-24T02:47:04,175 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,175 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,175 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,175 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424170) roll requested 2024-11-24T02:47:04,175 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424175 2024-11-24T02:47:04,180 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,180 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424175 2024-11-24T02:47:04,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,181 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424175) roll requested 2024-11-24T02:47:04,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424181 2024-11-24T02:47:04,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424175 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424181 2024-11-24T02:47:04,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,187 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424181) roll requested 2024-11-24T02:47:04,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424188 2024-11-24T02:47:04,194 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,194 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,194 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,194 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424181 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424188 2024-11-24T02:47:04,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,195 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424188) roll requested 2024-11-24T02:47:04,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424195 2024-11-24T02:47:04,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424195 2024-11-24T02:47:04,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,202 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424195) roll requested 2024-11-24T02:47:04,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424203 2024-11-24T02:47:04,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,209 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,209 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,209 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424195 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424203 2024-11-24T02:47:04,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,210 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424203) roll requested 2024-11-24T02:47:04,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424210 2024-11-24T02:47:04,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,217 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,217 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,217 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,217 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,217 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424203 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424210 2024-11-24T02:47:04,218 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,218 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,218 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,218 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424210) roll requested 2024-11-24T02:47:04,218 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424218 2024-11-24T02:47:04,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,227 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424210 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424218 2024-11-24T02:47:04,227 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,227 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,227 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,228 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424218) roll requested 2024-11-24T02:47:04,228 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424228 2024-11-24T02:47:04,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424218 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424228 2024-11-24T02:47:04,237 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,237 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424228) roll requested 2024-11-24T02:47:04,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424237 2024-11-24T02:47:04,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,246 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,246 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,246 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,246 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424228 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424237 2024-11-24T02:47:04,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424237) roll requested 2024-11-24T02:47:04,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424248 2024-11-24T02:47:04,253 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424237 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424248 2024-11-24T02:47:04,254 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,254 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,254 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,254 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424248) roll requested 2024-11-24T02:47:04,255 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424254 2024-11-24T02:47:04,259 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,259 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,259 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,259 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,259 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424248 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424254 2024-11-24T02:47:04,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424254) roll requested 2024-11-24T02:47:04,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424259 2024-11-24T02:47:04,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,264 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,264 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424254 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424259 2024-11-24T02:47:04,265 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,265 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424259) roll requested 2024-11-24T02:47:04,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424265 2024-11-24T02:47:04,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,270 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424265 2024-11-24T02:47:04,275 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,275 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,275 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,275 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424265) roll requested 2024-11-24T02:47:04,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424275 2024-11-24T02:47:04,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424265 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424275 2024-11-24T02:47:04,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,281 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424275) roll requested 2024-11-24T02:47:04,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424281 2024-11-24T02:47:04,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424275 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424281 2024-11-24T02:47:04,286 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,286 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,286 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,286 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424281) roll requested 2024-11-24T02:47:04,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424286 2024-11-24T02:47:04,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424281 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424286 2024-11-24T02:47:04,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,292 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424286) roll requested 2024-11-24T02:47:04,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424292 2024-11-24T02:47:04,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424286 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424292 2024-11-24T02:47:04,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,299 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424292) roll requested 2024-11-24T02:47:04,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424299 2024-11-24T02:47:04,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424292 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424299 2024-11-24T02:47:04,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,306 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424299) roll requested 2024-11-24T02:47:04,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424306 2024-11-24T02:47:04,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424306 2024-11-24T02:47:04,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,314 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424306) roll requested 2024-11-24T02:47:04,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424314 2024-11-24T02:47:04,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424306 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424314 2024-11-24T02:47:04,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,323 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424314) roll requested 2024-11-24T02:47:04,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424323 2024-11-24T02:47:04,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424314 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424323 2024-11-24T02:47:04,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,331 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424323) roll requested 2024-11-24T02:47:04,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424331 2024-11-24T02:47:04,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,339 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,339 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,339 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,339 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,339 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424323 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424331 2024-11-24T02:47:04,340 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,340 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,340 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,340 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424331) roll requested 2024-11-24T02:47:04,340 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424340 2024-11-24T02:47:04,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424331 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424340 2024-11-24T02:47:04,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,350 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424340) roll requested 2024-11-24T02:47:04,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424350 2024-11-24T02:47:04,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424340 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424350 2024-11-24T02:47:04,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424350) roll requested 2024-11-24T02:47:04,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424356 2024-11-24T02:47:04,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424350 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424356 2024-11-24T02:47:04,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,360 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424356) roll requested 2024-11-24T02:47:04,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424361 2024-11-24T02:47:04,365 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,365 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,365 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424361 2024-11-24T02:47:04,366 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,366 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,366 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424361) roll requested 2024-11-24T02:47:04,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424366 2024-11-24T02:47:04,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424361 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424366 2024-11-24T02:47:04,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424366) roll requested 2024-11-24T02:47:04,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424372 2024-11-24T02:47:04,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,378 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,378 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424366 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424372 2024-11-24T02:47:04,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424372) roll requested 2024-11-24T02:47:04,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424379 2024-11-24T02:47:04,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424379 2024-11-24T02:47:04,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424379) roll requested 2024-11-24T02:47:04,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424387 2024-11-24T02:47:04,394 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,395 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424387 2024-11-24T02:47:04,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,395 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424387) roll requested 2024-11-24T02:47:04,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424396 2024-11-24T02:47:04,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424396 2024-11-24T02:47:04,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424396) roll requested 2024-11-24T02:47:04,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424403 2024-11-24T02:47:04,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424396 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424403 2024-11-24T02:47:04,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424403) roll requested 2024-11-24T02:47:04,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424408 2024-11-24T02:47:04,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424408 2024-11-24T02:47:04,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,415 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424408) roll requested 2024-11-24T02:47:04,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424415 2024-11-24T02:47:04,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424415 2024-11-24T02:47:04,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424415) roll requested 2024-11-24T02:47:04,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424424 2024-11-24T02:47:04,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424415 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424424 2024-11-24T02:47:04,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424424) roll requested 2024-11-24T02:47:04,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424433 2024-11-24T02:47:04,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424433 2024-11-24T02:47:04,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,442 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424433) roll requested 2024-11-24T02:47:04,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424443 2024-11-24T02:47:04,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424433 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424443 2024-11-24T02:47:04,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,450 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424443) roll requested 2024-11-24T02:47:04,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424450 2024-11-24T02:47:04,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424443 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424450 2024-11-24T02:47:04,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,458 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424450) roll requested 2024-11-24T02:47:04,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424459 2024-11-24T02:47:04,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424459 2024-11-24T02:47:04,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,466 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424459) roll requested 2024-11-24T02:47:04,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424466 2024-11-24T02:47:04,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424466 2024-11-24T02:47:04,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,472 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424466) roll requested 2024-11-24T02:47:04,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424472 2024-11-24T02:47:04,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424466 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424472 2024-11-24T02:47:04,478 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,478 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,478 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424472) roll requested 2024-11-24T02:47:04,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424478 2024-11-24T02:47:04,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424472 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424478 2024-11-24T02:47:04,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,485 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424478) roll requested 2024-11-24T02:47:04,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424485 2024-11-24T02:47:04,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,492 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424478 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424485 2024-11-24T02:47:04,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424485) roll requested 2024-11-24T02:47:04,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424493 2024-11-24T02:47:04,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424485 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424493 2024-11-24T02:47:04,501 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,501 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424493) roll requested 2024-11-24T02:47:04,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424501 2024-11-24T02:47:04,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424493 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424501 2024-11-24T02:47:04,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,507 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424501) roll requested 2024-11-24T02:47:04,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424508 2024-11-24T02:47:04,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424501 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424508 2024-11-24T02:47:04,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424508) roll requested 2024-11-24T02:47:04,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424516 2024-11-24T02:47:04,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424508 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424516 2024-11-24T02:47:04,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,523 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424516) roll requested 2024-11-24T02:47:04,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424523 2024-11-24T02:47:04,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424516 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424523 2024-11-24T02:47:04,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424523) roll requested 2024-11-24T02:47:04,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424530 2024-11-24T02:47:04,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424530 2024-11-24T02:47:04,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424530) roll requested 2024-11-24T02:47:04,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424538 2024-11-24T02:47:04,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424538 2024-11-24T02:47:04,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,547 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424538) roll requested 2024-11-24T02:47:04,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424547 2024-11-24T02:47:04,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424547 2024-11-24T02:47:04,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424547) roll requested 2024-11-24T02:47:04,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424556 2024-11-24T02:47:04,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424556 2024-11-24T02:47:04,562 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,562 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424556) roll requested 2024-11-24T02:47:04,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424562 2024-11-24T02:47:04,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424562 2024-11-24T02:47:04,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424562) roll requested 2024-11-24T02:47:04,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424568 2024-11-24T02:47:04,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424562 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424568 2024-11-24T02:47:04,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,575 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424568) roll requested 2024-11-24T02:47:04,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424576 2024-11-24T02:47:04,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424576 2024-11-24T02:47:04,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424576) roll requested 2024-11-24T02:47:04,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424584 2024-11-24T02:47:04,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,592 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,592 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424576 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424584 2024-11-24T02:47:04,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,592 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424584) roll requested 2024-11-24T02:47:04,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424592 2024-11-24T02:47:04,598 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,598 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,598 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,598 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,598 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424584 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424592 2024-11-24T02:47:04,599 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,599 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,599 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,599 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424592) roll requested 2024-11-24T02:47:04,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424599 2024-11-24T02:47:04,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424592 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424599 2024-11-24T02:47:04,606 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,606 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,606 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,606 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424599) roll requested 2024-11-24T02:47:04,606 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424606 2024-11-24T02:47:04,612 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,612 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,612 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424599 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424606 2024-11-24T02:47:04,613 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,613 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424606) roll requested 2024-11-24T02:47:04,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424614 2024-11-24T02:47:04,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424606 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424614 2024-11-24T02:47:04,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,623 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424614) roll requested 2024-11-24T02:47:04,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424623 2024-11-24T02:47:04,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,632 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,632 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,632 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424614 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424623 2024-11-24T02:47:04,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,633 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424623) roll requested 2024-11-24T02:47:04,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424633 2024-11-24T02:47:04,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424623 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424633 2024-11-24T02:47:04,638 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,638 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,638 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,638 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424633) roll requested 2024-11-24T02:47:04,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424638 2024-11-24T02:47:04,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424633 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424638 2024-11-24T02:47:04,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,644 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424638) roll requested 2024-11-24T02:47:04,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424644 2024-11-24T02:47:04,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,650 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,650 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424644 2024-11-24T02:47:04,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,650 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,651 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424644) roll requested 2024-11-24T02:47:04,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424651 2024-11-24T02:47:04,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424644 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424651 2024-11-24T02:47:04,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,658 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424651) roll requested 2024-11-24T02:47:04,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424658 2024-11-24T02:47:04,665 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,665 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,665 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,665 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424651 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424658 2024-11-24T02:47:04,666 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,666 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,666 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424658) roll requested 2024-11-24T02:47:04,667 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424667 2024-11-24T02:47:04,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424658 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424667 2024-11-24T02:47:04,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,674 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,674 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424667) roll requested 2024-11-24T02:47:04,674 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424674 2024-11-24T02:47:04,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424667 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424674 2024-11-24T02:47:04,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424674) roll requested 2024-11-24T02:47:04,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424681 2024-11-24T02:47:04,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,689 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,689 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424674 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424681 2024-11-24T02:47:04,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424681) roll requested 2024-11-24T02:47:04,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424690 2024-11-24T02:47:04,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424690 2024-11-24T02:47:04,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,696 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424690) roll requested 2024-11-24T02:47:04,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424696 2024-11-24T02:47:04,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424690 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424696 2024-11-24T02:47:04,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,703 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424696) roll requested 2024-11-24T02:47:04,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424703 2024-11-24T02:47:04,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424696 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424703 2024-11-24T02:47:04,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,711 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424703) roll requested 2024-11-24T02:47:04,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424712 2024-11-24T02:47:04,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424703 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424712 2024-11-24T02:47:04,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,721 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424712) roll requested 2024-11-24T02:47:04,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424721 2024-11-24T02:47:04,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,726 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424712 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424721 2024-11-24T02:47:04,727 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,727 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,727 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,727 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424721) roll requested 2024-11-24T02:47:04,727 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424727 2024-11-24T02:47:04,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,734 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,734 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424721 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424727 2024-11-24T02:47:04,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424727) roll requested 2024-11-24T02:47:04,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424735 2024-11-24T02:47:04,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424727 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424735 2024-11-24T02:47:04,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,743 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424735) roll requested 2024-11-24T02:47:04,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424743 2024-11-24T02:47:04,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,749 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,749 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,749 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424743 2024-11-24T02:47:04,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,750 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424743) roll requested 2024-11-24T02:47:04,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424750 2024-11-24T02:47:04,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424743 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424750 2024-11-24T02:47:04,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424750) roll requested 2024-11-24T02:47:04,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424759 2024-11-24T02:47:04,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,765 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,765 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424750 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424759 2024-11-24T02:47:04,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,766 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424759) roll requested 2024-11-24T02:47:04,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424766 2024-11-24T02:47:04,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424766 2024-11-24T02:47:04,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424766) roll requested 2024-11-24T02:47:04,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424773 2024-11-24T02:47:04,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424766 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424773 2024-11-24T02:47:04,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424773) roll requested 2024-11-24T02:47:04,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424782 2024-11-24T02:47:04,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424782 2024-11-24T02:47:04,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,793 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424782) roll requested 2024-11-24T02:47:04,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424793 2024-11-24T02:47:04,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424782 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424793 2024-11-24T02:47:04,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424793) roll requested 2024-11-24T02:47:04,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424803 2024-11-24T02:47:04,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424793 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424803 2024-11-24T02:47:04,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,820 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424803) roll requested 2024-11-24T02:47:04,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424820 2024-11-24T02:47:04,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424803 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424820 2024-11-24T02:47:04,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424820) roll requested 2024-11-24T02:47:04,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424831 2024-11-24T02:47:04,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424820 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424831 2024-11-24T02:47:04,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,838 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,838 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,838 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424831) roll requested 2024-11-24T02:47:04,838 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424838 2024-11-24T02:47:04,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,844 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424831 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424838 2024-11-24T02:47:04,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424838) roll requested 2024-11-24T02:47:04,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424844 2024-11-24T02:47:04,852 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,852 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,852 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,852 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424838 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424844 2024-11-24T02:47:04,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424844) roll requested 2024-11-24T02:47:04,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424853 2024-11-24T02:47:04,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424853 2024-11-24T02:47:04,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,860 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424853) roll requested 2024-11-24T02:47:04,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424860 2024-11-24T02:47:04,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,867 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,867 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424853 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424860 2024-11-24T02:47:04,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,868 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,868 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424860) roll requested 2024-11-24T02:47:04,868 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424868 2024-11-24T02:47:04,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424860 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424868 2024-11-24T02:47:04,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424868) roll requested 2024-11-24T02:47:04,877 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424876 2024-11-24T02:47:04,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,882 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,882 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,882 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424868 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424876 2024-11-24T02:47:04,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,883 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424876) roll requested 2024-11-24T02:47:04,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424883 2024-11-24T02:47:04,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424883 2024-11-24T02:47:04,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,889 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424883) roll requested 2024-11-24T02:47:04,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424889 2024-11-24T02:47:04,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424883 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424889 2024-11-24T02:47:04,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,895 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424889) roll requested 2024-11-24T02:47:04,895 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424895 2024-11-24T02:47:04,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424895 2024-11-24T02:47:04,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424895) roll requested 2024-11-24T02:47:04,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424904 2024-11-24T02:47:04,909 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424895 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424904 2024-11-24T02:47:04,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,910 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,910 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424904) roll requested 2024-11-24T02:47:04,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424911 2024-11-24T02:47:04,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,917 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,917 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424911 2024-11-24T02:47:04,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424911) roll requested 2024-11-24T02:47:04,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424917 2024-11-24T02:47:04,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424911 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424917 2024-11-24T02:47:04,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,923 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,923 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424917) roll requested 2024-11-24T02:47:04,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424923 2024-11-24T02:47:04,931 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,931 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,931 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,931 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,931 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424917 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424923 2024-11-24T02:47:04,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,933 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424923) roll requested 2024-11-24T02:47:04,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424934 2024-11-24T02:47:04,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424923 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424934 2024-11-24T02:47:04,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424934) roll requested 2024-11-24T02:47:04,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424939 2024-11-24T02:47:04,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,949 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,949 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,949 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424934 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424939 2024-11-24T02:47:04,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,949 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424939) roll requested 2024-11-24T02:47:04,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424950 2024-11-24T02:47:04,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424950 2024-11-24T02:47:04,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,959 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424950) roll requested 2024-11-24T02:47:04,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424959 2024-11-24T02:47:04,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424950 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424959 2024-11-24T02:47:04,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,972 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424959) roll requested 2024-11-24T02:47:04,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424972 2024-11-24T02:47:04,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424959 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424972 2024-11-24T02:47:04,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,979 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,979 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424972) roll requested 2024-11-24T02:47:04,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424979 2024-11-24T02:47:04,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424972 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424979 2024-11-24T02:47:04,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,988 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424979) roll requested 2024-11-24T02:47:04,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424988 2024-11-24T02:47:04,992 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,992 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,992 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424979 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424988 2024-11-24T02:47:04,993 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,993 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424988) roll requested 2024-11-24T02:47:04,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424993 2024-11-24T02:47:04,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:04,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424993 2024-11-24T02:47:04,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:04,998 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:04,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424993) roll requested 2024-11-24T02:47:04,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416424999 2024-11-24T02:47:05,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424993 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424999 2024-11-24T02:47:05,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,004 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416424999) roll requested 2024-11-24T02:47:05,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425004 2024-11-24T02:47:05,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,008 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416424999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425004 2024-11-24T02:47:05,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425004) roll requested 2024-11-24T02:47:05,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425009 2024-11-24T02:47:05,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,014 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,014 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,014 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,014 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425004 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425009 2024-11-24T02:47:05,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,014 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425009) roll requested 2024-11-24T02:47:05,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425014 2024-11-24T02:47:05,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425009 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425014 2024-11-24T02:47:05,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425014) roll requested 2024-11-24T02:47:05,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425019 2024-11-24T02:47:05,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425019 2024-11-24T02:47:05,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425019) roll requested 2024-11-24T02:47:05,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425026 2024-11-24T02:47:05,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425019 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425026 2024-11-24T02:47:05,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,031 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425026) roll requested 2024-11-24T02:47:05,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425031 2024-11-24T02:47:05,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425026 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425031 2024-11-24T02:47:05,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425031) roll requested 2024-11-24T02:47:05,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425036 2024-11-24T02:47:05,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425031 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425036 2024-11-24T02:47:05,041 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,041 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,041 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,041 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425036) roll requested 2024-11-24T02:47:05,041 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425041 2024-11-24T02:47:05,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,046 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425041 2024-11-24T02:47:05,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425041) roll requested 2024-11-24T02:47:05,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425047 2024-11-24T02:47:05,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425041 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425047 2024-11-24T02:47:05,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425047) roll requested 2024-11-24T02:47:05,054 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425054 2024-11-24T02:47:05,061 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,061 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,061 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,061 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,061 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,061 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425054 2024-11-24T02:47:05,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,062 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,062 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425054) roll requested 2024-11-24T02:47:05,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425062 2024-11-24T02:47:05,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425054 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425062 2024-11-24T02:47:05,070 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,070 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425062) roll requested 2024-11-24T02:47:05,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425070 2024-11-24T02:47:05,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,077 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425062 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425070 2024-11-24T02:47:05,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425070) roll requested 2024-11-24T02:47:05,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425078 2024-11-24T02:47:05,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425070 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425078 2024-11-24T02:47:05,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425078) roll requested 2024-11-24T02:47:05,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425087 2024-11-24T02:47:05,095 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,095 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425087 2024-11-24T02:47:05,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,096 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425087) roll requested 2024-11-24T02:47:05,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425096 2024-11-24T02:47:05,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425087 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425096 2024-11-24T02:47:05,106 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,106 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,106 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425096) roll requested 2024-11-24T02:47:05,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425106 2024-11-24T02:47:05,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,115 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425106 2024-11-24T02:47:05,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,115 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425106) roll requested 2024-11-24T02:47:05,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425116 2024-11-24T02:47:05,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,122 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,122 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,122 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,122 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,122 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425106 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425116 2024-11-24T02:47:05,123 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,123 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,123 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425116) roll requested 2024-11-24T02:47:05,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425123 2024-11-24T02:47:05,128 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,128 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,128 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,128 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,128 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425116 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425123 2024-11-24T02:47:05,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,129 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425123) roll requested 2024-11-24T02:47:05,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425129 2024-11-24T02:47:05,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425123 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425129 2024-11-24T02:47:05,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425129) roll requested 2024-11-24T02:47:05,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425134 2024-11-24T02:47:05,139 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,139 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,139 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,139 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425129 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425134 2024-11-24T02:47:05,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,140 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,140 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,140 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425134) roll requested 2024-11-24T02:47:05,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425140 2024-11-24T02:47:05,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425140 2024-11-24T02:47:05,147 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,147 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425140) roll requested 2024-11-24T02:47:05,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425148 2024-11-24T02:47:05,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425140 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425148 2024-11-24T02:47:05,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425148) roll requested 2024-11-24T02:47:05,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425153 2024-11-24T02:47:05,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425148 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425153 2024-11-24T02:47:05,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,159 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425153) roll requested 2024-11-24T02:47:05,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425159 2024-11-24T02:47:05,166 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,166 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425159 2024-11-24T02:47:05,167 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,167 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,167 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,167 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425159) roll requested 2024-11-24T02:47:05,167 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425167 2024-11-24T02:47:05,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425159 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425167 2024-11-24T02:47:05,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,173 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425167) roll requested 2024-11-24T02:47:05,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425174 2024-11-24T02:47:05,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425167 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425174 2024-11-24T02:47:05,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425174) roll requested 2024-11-24T02:47:05,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425180 2024-11-24T02:47:05,185 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,185 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,185 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,185 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,185 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425174 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425180 2024-11-24T02:47:05,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,185 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,186 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425180) roll requested 2024-11-24T02:47:05,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425186 2024-11-24T02:47:05,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425186 2024-11-24T02:47:05,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,196 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425186) roll requested 2024-11-24T02:47:05,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425197 2024-11-24T02:47:05,203 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425186 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425197 2024-11-24T02:47:05,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,208 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425197) roll requested 2024-11-24T02:47:05,209 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425208 2024-11-24T02:47:05,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,226 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425208 2024-11-24T02:47:05,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,226 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425208) roll requested 2024-11-24T02:47:05,227 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425226 2024-11-24T02:47:05,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425208 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425226 2024-11-24T02:47:05,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,240 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425226) roll requested 2024-11-24T02:47:05,240 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425240 2024-11-24T02:47:05,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425226 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425240 2024-11-24T02:47:05,248 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,248 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,248 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,248 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425240) roll requested 2024-11-24T02:47:05,249 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425249 2024-11-24T02:47:05,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,254 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425240 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425249 2024-11-24T02:47:05,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,256 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425249) roll requested 2024-11-24T02:47:05,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425256 2024-11-24T02:47:05,261 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,261 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425249 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425256 2024-11-24T02:47:05,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,262 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425256) roll requested 2024-11-24T02:47:05,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425262 2024-11-24T02:47:05,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425256 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425262 2024-11-24T02:47:05,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425262) roll requested 2024-11-24T02:47:05,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425267 2024-11-24T02:47:05,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425262 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425267 2024-11-24T02:47:05,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,272 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425267) roll requested 2024-11-24T02:47:05,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425272 2024-11-24T02:47:05,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425267 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425272 2024-11-24T02:47:05,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,280 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425272) roll requested 2024-11-24T02:47:05,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425280 2024-11-24T02:47:05,286 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,286 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,286 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,286 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425272 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425280 2024-11-24T02:47:05,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425280) roll requested 2024-11-24T02:47:05,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425287 2024-11-24T02:47:05,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,295 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,295 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425280 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425287 2024-11-24T02:47:05,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,296 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,296 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425287) roll requested 2024-11-24T02:47:05,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425297 2024-11-24T02:47:05,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425297 2024-11-24T02:47:05,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,304 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425297) roll requested 2024-11-24T02:47:05,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425305 2024-11-24T02:47:05,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,312 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,312 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,312 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,312 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425297 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425305 2024-11-24T02:47:05,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,313 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,313 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425305) roll requested 2024-11-24T02:47:05,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425313 2024-11-24T02:47:05,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,319 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425313 2024-11-24T02:47:05,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,320 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,320 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425313) roll requested 2024-11-24T02:47:05,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425320 2024-11-24T02:47:05,326 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,326 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,326 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,326 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,326 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,326 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425313 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425320 2024-11-24T02:47:05,327 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,327 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,327 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,327 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425320) roll requested 2024-11-24T02:47:05,327 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425327 2024-11-24T02:47:05,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,333 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,333 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,333 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,333 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425320 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425327 2024-11-24T02:47:05,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,334 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,334 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425327) roll requested 2024-11-24T02:47:05,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425334 2024-11-24T02:47:05,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425327 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425334 2024-11-24T02:47:05,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,342 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425334) roll requested 2024-11-24T02:47:05,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425342 2024-11-24T02:47:05,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425334 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425342 2024-11-24T02:47:05,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,349 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425342) roll requested 2024-11-24T02:47:05,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425349 2024-11-24T02:47:05,354 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425342 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425349 2024-11-24T02:47:05,355 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,355 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425349) roll requested 2024-11-24T02:47:05,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425355 2024-11-24T02:47:05,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,362 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,362 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,362 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,362 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425349 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425355 2024-11-24T02:47:05,363 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,363 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,363 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,363 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425355) roll requested 2024-11-24T02:47:05,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425363 2024-11-24T02:47:05,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425355 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425363 2024-11-24T02:47:05,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,372 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425363) roll requested 2024-11-24T02:47:05,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425372 2024-11-24T02:47:05,378 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,379 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,379 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,379 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,379 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425363 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425372 2024-11-24T02:47:05,379 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,380 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,380 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,380 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425372) roll requested 2024-11-24T02:47:05,380 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425380 2024-11-24T02:47:05,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,386 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425372 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425380 2024-11-24T02:47:05,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,387 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,387 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425380) roll requested 2024-11-24T02:47:05,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425387 2024-11-24T02:47:05,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425380 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425387 2024-11-24T02:47:05,393 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,393 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,393 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425387) roll requested 2024-11-24T02:47:05,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425393 2024-11-24T02:47:05,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425387 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425393 2024-11-24T02:47:05,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425393) roll requested 2024-11-24T02:47:05,401 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425400 2024-11-24T02:47:05,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425400 2024-11-24T02:47:05,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,408 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425400) roll requested 2024-11-24T02:47:05,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425408 2024-11-24T02:47:05,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425408 2024-11-24T02:47:05,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,416 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425408) roll requested 2024-11-24T02:47:05,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425416 2024-11-24T02:47:05,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,423 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,423 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425408 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425416 2024-11-24T02:47:05,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,424 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425416) roll requested 2024-11-24T02:47:05,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425424 2024-11-24T02:47:05,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425416 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425424 2024-11-24T02:47:05,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,432 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,432 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,432 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425424) roll requested 2024-11-24T02:47:05,432 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425432 2024-11-24T02:47:05,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425424 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425432 2024-11-24T02:47:05,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,439 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425432) roll requested 2024-11-24T02:47:05,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425439 2024-11-24T02:47:05,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425432 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425439 2024-11-24T02:47:05,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,448 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425439) roll requested 2024-11-24T02:47:05,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425448 2024-11-24T02:47:05,453 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,453 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,453 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,453 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425439 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425448 2024-11-24T02:47:05,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,453 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,453 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425448) roll requested 2024-11-24T02:47:05,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425453 2024-11-24T02:47:05,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,458 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425448 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425453 2024-11-24T02:47:05,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,459 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425453) roll requested 2024-11-24T02:47:05,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425459 2024-11-24T02:47:05,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425453 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425459 2024-11-24T02:47:05,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425459) roll requested 2024-11-24T02:47:05,464 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425464 2024-11-24T02:47:05,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425459 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425464 2024-11-24T02:47:05,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,469 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425464) roll requested 2024-11-24T02:47:05,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425470 2024-11-24T02:47:05,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425470 2024-11-24T02:47:05,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,476 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425470) roll requested 2024-11-24T02:47:05,476 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425476 2024-11-24T02:47:05,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,483 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,483 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,483 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425470 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425476 2024-11-24T02:47:05,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,484 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425476) roll requested 2024-11-24T02:47:05,484 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425484 2024-11-24T02:47:05,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425476 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425484 2024-11-24T02:47:05,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,492 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425484) roll requested 2024-11-24T02:47:05,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425492 2024-11-24T02:47:05,500 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,500 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,500 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,500 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425484 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425492 2024-11-24T02:47:05,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,503 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425492) roll requested 2024-11-24T02:47:05,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425504 2024-11-24T02:47:05,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,516 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425492 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425504 2024-11-24T02:47:05,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425504) roll requested 2024-11-24T02:47:05,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425520 2024-11-24T02:47:05,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425504 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425520 2024-11-24T02:47:05,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425520) roll requested 2024-11-24T02:47:05,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425526 2024-11-24T02:47:05,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425526 2024-11-24T02:47:05,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425526) roll requested 2024-11-24T02:47:05,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425531 2024-11-24T02:47:05,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,535 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425526 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425531 2024-11-24T02:47:05,535 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,535 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,535 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,536 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425531) roll requested 2024-11-24T02:47:05,536 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425536 2024-11-24T02:47:05,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,540 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,540 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,540 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425536 2024-11-24T02:47:05,541 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,541 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425536) roll requested 2024-11-24T02:47:05,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425541 2024-11-24T02:47:05,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425536 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425541 2024-11-24T02:47:05,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,552 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425541) roll requested 2024-11-24T02:47:05,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425552 2024-11-24T02:47:05,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425541 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425552 2024-11-24T02:47:05,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,560 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,560 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425552) roll requested 2024-11-24T02:47:05,560 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425560 2024-11-24T02:47:05,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425552 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425560 2024-11-24T02:47:05,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,566 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,566 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425560) roll requested 2024-11-24T02:47:05,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425567 2024-11-24T02:47:05,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425560 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425567 2024-11-24T02:47:05,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425567) roll requested 2024-11-24T02:47:05,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425575 2024-11-24T02:47:05,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425567 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425575 2024-11-24T02:47:05,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,583 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425575) roll requested 2024-11-24T02:47:05,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425583 2024-11-24T02:47:05,588 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,588 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425575 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425583 2024-11-24T02:47:05,589 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,589 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425583) roll requested 2024-11-24T02:47:05,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425589 2024-11-24T02:47:05,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,595 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425583 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425589 2024-11-24T02:47:05,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,596 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425589) roll requested 2024-11-24T02:47:05,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425596 2024-11-24T02:47:05,604 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,604 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,604 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,604 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,604 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,604 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425589 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425596 2024-11-24T02:47:05,605 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,605 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,605 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,605 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425596) roll requested 2024-11-24T02:47:05,605 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425605 2024-11-24T02:47:05,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425605 2024-11-24T02:47:05,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425605) roll requested 2024-11-24T02:47:05,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425610 2024-11-24T02:47:05,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,614 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425605 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425610 2024-11-24T02:47:05,615 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,615 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,615 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,615 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425610) roll requested 2024-11-24T02:47:05,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425615 2024-11-24T02:47:05,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425610 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425615 2024-11-24T02:47:05,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,628 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425615) roll requested 2024-11-24T02:47:05,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425628 2024-11-24T02:47:05,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425615 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425628 2024-11-24T02:47:05,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,656 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425628) roll requested 2024-11-24T02:47:05,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425656 2024-11-24T02:47:05,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425628 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425656 2024-11-24T02:47:05,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,662 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425656) roll requested 2024-11-24T02:47:05,662 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425662 2024-11-24T02:47:05,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425656 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425662 2024-11-24T02:47:05,668 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,668 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425662) roll requested 2024-11-24T02:47:05,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425668 2024-11-24T02:47:05,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,673 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,673 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,673 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,673 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,673 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425662 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425668 2024-11-24T02:47:05,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,674 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425668) roll requested 2024-11-24T02:47:05,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425675 2024-11-24T02:47:05,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,681 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,681 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425668 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425675 2024-11-24T02:47:05,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425675) roll requested 2024-11-24T02:47:05,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425681 2024-11-24T02:47:05,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425675 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425681 2024-11-24T02:47:05,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,689 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425681) roll requested 2024-11-24T02:47:05,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425689 2024-11-24T02:47:05,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,694 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,694 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425689 2024-11-24T02:47:05,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,694 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425689) roll requested 2024-11-24T02:47:05,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425695 2024-11-24T02:47:05,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,699 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,699 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425689 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425695 2024-11-24T02:47:05,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,700 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425695) roll requested 2024-11-24T02:47:05,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425700 2024-11-24T02:47:05,705 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425695 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425700 2024-11-24T02:47:05,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,706 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425700) roll requested 2024-11-24T02:47:05,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425706 2024-11-24T02:47:05,711 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425700 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425706 2024-11-24T02:47:05,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,713 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,713 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425706) roll requested 2024-11-24T02:47:05,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425713 2024-11-24T02:47:05,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,721 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425706 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425713 2024-11-24T02:47:05,722 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,722 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,722 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425713) roll requested 2024-11-24T02:47:05,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425722 2024-11-24T02:47:05,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425713 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425722 2024-11-24T02:47:05,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,729 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,729 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425722) roll requested 2024-11-24T02:47:05,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425730 2024-11-24T02:47:05,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,736 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,736 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,736 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425722 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425730 2024-11-24T02:47:05,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,736 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425730) roll requested 2024-11-24T02:47:05,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425737 2024-11-24T02:47:05,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,744 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425737 2024-11-24T02:47:05,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425737) roll requested 2024-11-24T02:47:05,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425745 2024-11-24T02:47:05,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425745 2024-11-24T02:47:05,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425745) roll requested 2024-11-24T02:47:05,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425752 2024-11-24T02:47:05,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425752 2024-11-24T02:47:05,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,764 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425752) roll requested 2024-11-24T02:47:05,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425764 2024-11-24T02:47:05,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425764 2024-11-24T02:47:05,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,773 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425764) roll requested 2024-11-24T02:47:05,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425773 2024-11-24T02:47:05,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425764 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425773 2024-11-24T02:47:05,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,778 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425773) roll requested 2024-11-24T02:47:05,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425779 2024-11-24T02:47:05,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,783 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425779 2024-11-24T02:47:05,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,784 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,784 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425779) roll requested 2024-11-24T02:47:05,784 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425784 2024-11-24T02:47:05,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425779 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425784 2024-11-24T02:47:05,791 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,791 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,791 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,791 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425784) roll requested 2024-11-24T02:47:05,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425791 2024-11-24T02:47:05,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425784 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425791 2024-11-24T02:47:05,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425791) roll requested 2024-11-24T02:47:05,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425796 2024-11-24T02:47:05,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425791 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425796 2024-11-24T02:47:05,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,802 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425796) roll requested 2024-11-24T02:47:05,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425802 2024-11-24T02:47:05,808 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,808 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,808 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,808 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425802 2024-11-24T02:47:05,809 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,809 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,809 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,809 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425802) roll requested 2024-11-24T02:47:05,809 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425809 2024-11-24T02:47:05,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425802 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425809 2024-11-24T02:47:05,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425809) roll requested 2024-11-24T02:47:05,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425816 2024-11-24T02:47:05,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425809 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425816 2024-11-24T02:47:05,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,823 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425816) roll requested 2024-11-24T02:47:05,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425823 2024-11-24T02:47:05,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425816 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425823 2024-11-24T02:47:05,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425823) roll requested 2024-11-24T02:47:05,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425836 2024-11-24T02:47:05,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425836 2024-11-24T02:47:05,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,842 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425836) roll requested 2024-11-24T02:47:05,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425842 2024-11-24T02:47:05,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,847 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,847 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,847 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425842 2024-11-24T02:47:05,848 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,848 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,848 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,848 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425842) roll requested 2024-11-24T02:47:05,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425848 2024-11-24T02:47:05,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425842 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425848 2024-11-24T02:47:05,854 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425848) roll requested 2024-11-24T02:47:05,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425855 2024-11-24T02:47:05,862 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,862 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,862 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,862 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425848 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425855 2024-11-24T02:47:05,863 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,863 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425855) roll requested 2024-11-24T02:47:05,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425863 2024-11-24T02:47:05,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425863 2024-11-24T02:47:05,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,870 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425863) roll requested 2024-11-24T02:47:05,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425870 2024-11-24T02:47:05,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,874 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425870 2024-11-24T02:47:05,876 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,876 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,876 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425870) roll requested 2024-11-24T02:47:05,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425876 2024-11-24T02:47:05,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425870 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425876 2024-11-24T02:47:05,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,882 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425876) roll requested 2024-11-24T02:47:05,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425882 2024-11-24T02:47:05,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425876 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425882 2024-11-24T02:47:05,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,888 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425882) roll requested 2024-11-24T02:47:05,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425889 2024-11-24T02:47:05,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425882 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425889 2024-11-24T02:47:05,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,895 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425889) roll requested 2024-11-24T02:47:05,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425896 2024-11-24T02:47:05,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425889 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425896 2024-11-24T02:47:05,903 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,903 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,903 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425896) roll requested 2024-11-24T02:47:05,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425904 2024-11-24T02:47:05,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425896 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425904 2024-11-24T02:47:05,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,912 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,912 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,912 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425904) roll requested 2024-11-24T02:47:05,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425912 2024-11-24T02:47:05,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425904 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425912 2024-11-24T02:47:05,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,922 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425912) roll requested 2024-11-24T02:47:05,923 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425922 2024-11-24T02:47:05,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,929 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,929 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,929 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425912 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425922 2024-11-24T02:47:05,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,930 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,930 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,930 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425922) roll requested 2024-11-24T02:47:05,930 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425930 2024-11-24T02:47:05,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425922 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425930 2024-11-24T02:47:05,935 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,935 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425930) roll requested 2024-11-24T02:47:05,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425935 2024-11-24T02:47:05,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425930 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425935 2024-11-24T02:47:05,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,940 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,940 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425935) roll requested 2024-11-24T02:47:05,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425940 2024-11-24T02:47:05,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425940 2024-11-24T02:47:05,947 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,947 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,947 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425940) roll requested 2024-11-24T02:47:05,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425947 2024-11-24T02:47:05,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,952 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425940 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425947 2024-11-24T02:47:05,953 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,953 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,953 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425947) roll requested 2024-11-24T02:47:05,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425954 2024-11-24T02:47:05,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425947 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425954 2024-11-24T02:47:05,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,961 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425954) roll requested 2024-11-24T02:47:05,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425961 2024-11-24T02:47:05,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,968 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,968 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425954 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425961 2024-11-24T02:47:05,969 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,969 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,969 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,969 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425961) roll requested 2024-11-24T02:47:05,969 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425969 2024-11-24T02:47:05,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425961 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425969 2024-11-24T02:47:05,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,977 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425969) roll requested 2024-11-24T02:47:05,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425977 2024-11-24T02:47:05,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,986 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,986 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425969 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425977 2024-11-24T02:47:05,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,987 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425977) roll requested 2024-11-24T02:47:05,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425988 2024-11-24T02:47:05,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:05,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425977 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425988 2024-11-24T02:47:05,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:05,995 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:05,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425988) roll requested 2024-11-24T02:47:05,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416425995 2024-11-24T02:47:06,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425988 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425995 2024-11-24T02:47:06,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,000 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416425995) roll requested 2024-11-24T02:47:06,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426001 2024-11-24T02:47:06,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,007 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,007 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,007 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,007 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416425995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426001 2024-11-24T02:47:06,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,008 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,008 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,008 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426001) roll requested 2024-11-24T02:47:06,008 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426008 2024-11-24T02:47:06,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426001 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426008 2024-11-24T02:47:06,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,013 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426008) roll requested 2024-11-24T02:47:06,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426014 2024-11-24T02:47:06,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426008 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426014 2024-11-24T02:47:06,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,019 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426014) roll requested 2024-11-24T02:47:06,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426020 2024-11-24T02:47:06,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,024 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426014 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426020 2024-11-24T02:47:06,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,025 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426020) roll requested 2024-11-24T02:47:06,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426025 2024-11-24T02:47:06,029 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,029 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,029 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426020 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426025 2024-11-24T02:47:06,030 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,030 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426025) roll requested 2024-11-24T02:47:06,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426030 2024-11-24T02:47:06,035 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,035 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,035 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426025 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426030 2024-11-24T02:47:06,036 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,036 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426030) roll requested 2024-11-24T02:47:06,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426036 2024-11-24T02:47:06,040 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,040 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,040 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,040 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,040 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,040 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426030 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426036 2024-11-24T02:47:06,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,042 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426036) roll requested 2024-11-24T02:47:06,042 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426042 2024-11-24T02:47:06,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426036 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426042 2024-11-24T02:47:06,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,047 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426042) roll requested 2024-11-24T02:47:06,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426047 2024-11-24T02:47:06,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426042 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426047 2024-11-24T02:47:06,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,054 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426047) roll requested 2024-11-24T02:47:06,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426055 2024-11-24T02:47:06,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426047 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426055 2024-11-24T02:47:06,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,064 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426055) roll requested 2024-11-24T02:47:06,065 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426064 2024-11-24T02:47:06,070 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,071 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,071 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,071 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,071 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,071 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426055 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426064 2024-11-24T02:47:06,071 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,071 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,071 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,072 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426064) roll requested 2024-11-24T02:47:06,072 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426072 2024-11-24T02:47:06,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,077 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,077 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426064 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426072 2024-11-24T02:47:06,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,078 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,078 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426072) roll requested 2024-11-24T02:47:06,078 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426078 2024-11-24T02:47:06,084 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,084 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,084 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,084 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,084 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426072 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426078 2024-11-24T02:47:06,085 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,085 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,085 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426078) roll requested 2024-11-24T02:47:06,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426086 2024-11-24T02:47:06,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426078 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426086 2024-11-24T02:47:06,095 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,095 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426086) roll requested 2024-11-24T02:47:06,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426096 2024-11-24T02:47:06,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426086 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426096 2024-11-24T02:47:06,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,102 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426096) roll requested 2024-11-24T02:47:06,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426103 2024-11-24T02:47:06,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426096 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426103 2024-11-24T02:47:06,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,107 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426103) roll requested 2024-11-24T02:47:06,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426107 2024-11-24T02:47:06,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426103 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426107 2024-11-24T02:47:06,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,114 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426107) roll requested 2024-11-24T02:47:06,114 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426114 2024-11-24T02:47:06,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426107 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426114 2024-11-24T02:47:06,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,121 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426114) roll requested 2024-11-24T02:47:06,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426121 2024-11-24T02:47:06,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426114 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426121 2024-11-24T02:47:06,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,128 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426121) roll requested 2024-11-24T02:47:06,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426128 2024-11-24T02:47:06,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426121 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426128 2024-11-24T02:47:06,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,135 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426128) roll requested 2024-11-24T02:47:06,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426135 2024-11-24T02:47:06,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,141 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426128 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426135 2024-11-24T02:47:06,141 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,141 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426135) roll requested 2024-11-24T02:47:06,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426141 2024-11-24T02:47:06,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426135 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426141 2024-11-24T02:47:06,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,148 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426141) roll requested 2024-11-24T02:47:06,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426149 2024-11-24T02:47:06,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,157 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,157 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,157 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,157 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426141 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426149 2024-11-24T02:47:06,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,158 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,158 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426149) roll requested 2024-11-24T02:47:06,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426158 2024-11-24T02:47:06,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,165 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,165 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,165 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,165 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,165 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426149 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426158 2024-11-24T02:47:06,166 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,166 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,166 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,166 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426158) roll requested 2024-11-24T02:47:06,166 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426166 2024-11-24T02:47:06,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426158 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426166 2024-11-24T02:47:06,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,172 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426166) roll requested 2024-11-24T02:47:06,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426172 2024-11-24T02:47:06,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426166 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426172 2024-11-24T02:47:06,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426172) roll requested 2024-11-24T02:47:06,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426180 2024-11-24T02:47:06,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426172 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426180 2024-11-24T02:47:06,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426180) roll requested 2024-11-24T02:47:06,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426188 2024-11-24T02:47:06,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426188 2024-11-24T02:47:06,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426188) roll requested 2024-11-24T02:47:06,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426197 2024-11-24T02:47:06,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426197 2024-11-24T02:47:06,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,204 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426197) roll requested 2024-11-24T02:47:06,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426204 2024-11-24T02:47:06,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426204 2024-11-24T02:47:06,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426204) roll requested 2024-11-24T02:47:06,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426212 2024-11-24T02:47:06,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426212 2024-11-24T02:47:06,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426212) roll requested 2024-11-24T02:47:06,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426220 2024-11-24T02:47:06,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,227 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,227 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,227 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426220 2024-11-24T02:47:06,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426220) roll requested 2024-11-24T02:47:06,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426229 2024-11-24T02:47:06,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426229 2024-11-24T02:47:06,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,238 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426229) roll requested 2024-11-24T02:47:06,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426238 2024-11-24T02:47:06,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426238 2024-11-24T02:47:06,245 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,245 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426238) roll requested 2024-11-24T02:47:06,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426245 2024-11-24T02:47:06,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,251 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,251 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426238 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426245 2024-11-24T02:47:06,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,251 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426245) roll requested 2024-11-24T02:47:06,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426251 2024-11-24T02:47:06,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426245 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426251 2024-11-24T02:47:06,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426251) roll requested 2024-11-24T02:47:06,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426259 2024-11-24T02:47:06,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426251 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426259 2024-11-24T02:47:06,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426259) roll requested 2024-11-24T02:47:06,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426268 2024-11-24T02:47:06,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,276 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426268 2024-11-24T02:47:06,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,276 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426268) roll requested 2024-11-24T02:47:06,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426277 2024-11-24T02:47:06,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426277 2024-11-24T02:47:06,283 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,283 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426277) roll requested 2024-11-24T02:47:06,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426284 2024-11-24T02:47:06,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426284 2024-11-24T02:47:06,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,291 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426284) roll requested 2024-11-24T02:47:06,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426291 2024-11-24T02:47:06,297 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,297 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,297 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426284 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426291 2024-11-24T02:47:06,298 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,298 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,298 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426291) roll requested 2024-11-24T02:47:06,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426299 2024-11-24T02:47:06,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,304 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426291 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426299 2024-11-24T02:47:06,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,305 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426299) roll requested 2024-11-24T02:47:06,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426305 2024-11-24T02:47:06,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426299 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426305 2024-11-24T02:47:06,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,311 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426305) roll requested 2024-11-24T02:47:06,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426311 2024-11-24T02:47:06,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,316 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,316 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,316 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426305 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426311 2024-11-24T02:47:06,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,317 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,317 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426311) roll requested 2024-11-24T02:47:06,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426317 2024-11-24T02:47:06,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426311 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426317 2024-11-24T02:47:06,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,322 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426317) roll requested 2024-11-24T02:47:06,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426322 2024-11-24T02:47:06,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426317 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426322 2024-11-24T02:47:06,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,328 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426322) roll requested 2024-11-24T02:47:06,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426328 2024-11-24T02:47:06,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,335 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426322 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426328 2024-11-24T02:47:06,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,335 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426328) roll requested 2024-11-24T02:47:06,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426336 2024-11-24T02:47:06,342 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,342 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,343 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,343 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,343 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426328 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426336 2024-11-24T02:47:06,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,348 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,348 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426336) roll requested 2024-11-24T02:47:06,348 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426348 2024-11-24T02:47:06,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426336 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426348 2024-11-24T02:47:06,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,356 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426348) roll requested 2024-11-24T02:47:06,356 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426356 2024-11-24T02:47:06,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426348 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426356 2024-11-24T02:47:06,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,362 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426356) roll requested 2024-11-24T02:47:06,362 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426362 2024-11-24T02:47:06,369 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,369 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,369 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,369 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,369 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,369 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426356 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426362 2024-11-24T02:47:06,370 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,370 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,370 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,370 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426362) roll requested 2024-11-24T02:47:06,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426370 2024-11-24T02:47:06,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426362 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426370 2024-11-24T02:47:06,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426370) roll requested 2024-11-24T02:47:06,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426378 2024-11-24T02:47:06,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,384 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,384 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,384 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,384 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426370 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426378 2024-11-24T02:47:06,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,384 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426378) roll requested 2024-11-24T02:47:06,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426385 2024-11-24T02:47:06,391 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,391 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,391 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,391 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426378 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426385 2024-11-24T02:47:06,392 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,392 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,392 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,393 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426385) roll requested 2024-11-24T02:47:06,393 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426393 2024-11-24T02:47:06,399 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,400 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,400 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,400 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,400 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,400 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426385 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426393 2024-11-24T02:47:06,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,400 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426393) roll requested 2024-11-24T02:47:06,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426400 2024-11-24T02:47:06,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426393 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426400 2024-11-24T02:47:06,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,405 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,406 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,406 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426400) roll requested 2024-11-24T02:47:06,406 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426406 2024-11-24T02:47:06,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426400 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426406 2024-11-24T02:47:06,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,411 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426406) roll requested 2024-11-24T02:47:06,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426411 2024-11-24T02:47:06,417 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,418 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,418 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,418 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,418 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,418 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426406 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426411 2024-11-24T02:47:06,418 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,418 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,419 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,419 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426411) roll requested 2024-11-24T02:47:06,419 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426419 2024-11-24T02:47:06,425 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426411 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426419 2024-11-24T02:47:06,426 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,426 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,426 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426419) roll requested 2024-11-24T02:47:06,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426426 2024-11-24T02:47:06,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426419 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426426 2024-11-24T02:47:06,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,434 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426426) roll requested 2024-11-24T02:47:06,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426434 2024-11-24T02:47:06,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,443 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426426 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426434 2024-11-24T02:47:06,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,445 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426434) roll requested 2024-11-24T02:47:06,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426445 2024-11-24T02:47:06,454 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,454 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,454 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,454 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,454 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,454 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426434 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426445 2024-11-24T02:47:06,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,455 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,455 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426445) roll requested 2024-11-24T02:47:06,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426456 2024-11-24T02:47:06,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426445 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426456 2024-11-24T02:47:06,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,465 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426456) roll requested 2024-11-24T02:47:06,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426465 2024-11-24T02:47:06,474 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,474 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426456 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426465 2024-11-24T02:47:06,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,475 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426465) roll requested 2024-11-24T02:47:06,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426475 2024-11-24T02:47:06,480 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,480 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,480 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,480 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,480 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,480 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426465 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426475 2024-11-24T02:47:06,481 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,481 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,481 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,481 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426475) roll requested 2024-11-24T02:47:06,481 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426481 2024-11-24T02:47:06,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,487 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426475 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426481 2024-11-24T02:47:06,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426481) roll requested 2024-11-24T02:47:06,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426487 2024-11-24T02:47:06,493 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426481 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426487 2024-11-24T02:47:06,494 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,494 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,494 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426487) roll requested 2024-11-24T02:47:06,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426494 2024-11-24T02:47:06,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,499 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426494 2024-11-24T02:47:06,499 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,499 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,499 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,499 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426494) roll requested 2024-11-24T02:47:06,500 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426500 2024-11-24T02:47:06,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426494 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426500 2024-11-24T02:47:06,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426500) roll requested 2024-11-24T02:47:06,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426505 2024-11-24T02:47:06,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426500 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426505 2024-11-24T02:47:06,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,510 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426505) roll requested 2024-11-24T02:47:06,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426510 2024-11-24T02:47:06,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,514 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426510 2024-11-24T02:47:06,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,515 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,515 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426510) roll requested 2024-11-24T02:47:06,515 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426515 2024-11-24T02:47:06,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426510 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426515 2024-11-24T02:47:06,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,520 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426515) roll requested 2024-11-24T02:47:06,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426520 2024-11-24T02:47:06,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,525 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,525 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426515 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426520 2024-11-24T02:47:06,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,525 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426520) roll requested 2024-11-24T02:47:06,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426525 2024-11-24T02:47:06,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426520 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426525 2024-11-24T02:47:06,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,531 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426525) roll requested 2024-11-24T02:47:06,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426531 2024-11-24T02:47:06,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426525 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426531 2024-11-24T02:47:06,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,538 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426531) roll requested 2024-11-24T02:47:06,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426538 2024-11-24T02:47:06,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426531 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426538 2024-11-24T02:47:06,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426538) roll requested 2024-11-24T02:47:06,547 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426547 2024-11-24T02:47:06,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426538 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426547 2024-11-24T02:47:06,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,556 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426547) roll requested 2024-11-24T02:47:06,557 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426556 2024-11-24T02:47:06,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426547 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426556 2024-11-24T02:47:06,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,563 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426556) roll requested 2024-11-24T02:47:06,563 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426563 2024-11-24T02:47:06,567 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426556 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426563 2024-11-24T02:47:06,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,568 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426563) roll requested 2024-11-24T02:47:06,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426568 2024-11-24T02:47:06,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,573 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,573 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,573 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,573 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426563 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426568 2024-11-24T02:47:06,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,574 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,574 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,574 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426568) roll requested 2024-11-24T02:47:06,574 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426574 2024-11-24T02:47:06,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,580 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,580 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426568 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426574 2024-11-24T02:47:06,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,580 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426574) roll requested 2024-11-24T02:47:06,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426581 2024-11-24T02:47:06,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,586 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,586 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426574 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426581 2024-11-24T02:47:06,587 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,587 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,587 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,587 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426581) roll requested 2024-11-24T02:47:06,587 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426587 2024-11-24T02:47:06,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,595 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,595 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,595 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,595 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426581 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426587 2024-11-24T02:47:06,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,595 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426587) roll requested 2024-11-24T02:47:06,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426596 2024-11-24T02:47:06,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426587 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426596 2024-11-24T02:47:06,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,603 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,603 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426596) roll requested 2024-11-24T02:47:06,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426603 2024-11-24T02:47:06,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426596 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426603 2024-11-24T02:47:06,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,610 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426603) roll requested 2024-11-24T02:47:06,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426611 2024-11-24T02:47:06,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,618 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426611 2024-11-24T02:47:06,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,619 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426611) roll requested 2024-11-24T02:47:06,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426619 2024-11-24T02:47:06,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426611 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426619 2024-11-24T02:47:06,629 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,629 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,629 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,629 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426619) roll requested 2024-11-24T02:47:06,629 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426629 2024-11-24T02:47:06,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,636 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,636 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,636 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,636 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,636 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426619 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426629 2024-11-24T02:47:06,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426629) roll requested 2024-11-24T02:47:06,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426637 2024-11-24T02:47:06,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,645 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,645 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,645 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,645 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,645 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426629 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426637 2024-11-24T02:47:06,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,646 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,646 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,646 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426637) roll requested 2024-11-24T02:47:06,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426646 2024-11-24T02:47:06,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426637 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426646 2024-11-24T02:47:06,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,654 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426646) roll requested 2024-11-24T02:47:06,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426654 2024-11-24T02:47:06,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426654 2024-11-24T02:47:06,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,662 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426654) roll requested 2024-11-24T02:47:06,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426663 2024-11-24T02:47:06,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426663 2024-11-24T02:47:06,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,670 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426663) roll requested 2024-11-24T02:47:06,671 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426671 2024-11-24T02:47:06,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426671 2024-11-24T02:47:06,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,678 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426671) roll requested 2024-11-24T02:47:06,679 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426678 2024-11-24T02:47:06,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426671 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426678 2024-11-24T02:47:06,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,685 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426678) roll requested 2024-11-24T02:47:06,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426686 2024-11-24T02:47:06,691 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426678 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426686 2024-11-24T02:47:06,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,692 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426686) roll requested 2024-11-24T02:47:06,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426693 2024-11-24T02:47:06,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426686 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426693 2024-11-24T02:47:06,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,701 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426693) roll requested 2024-11-24T02:47:06,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426701 2024-11-24T02:47:06,708 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,708 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426693 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426701 2024-11-24T02:47:06,709 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,709 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426701) roll requested 2024-11-24T02:47:06,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426710 2024-11-24T02:47:06,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426701 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426710 2024-11-24T02:47:06,718 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,718 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426710) roll requested 2024-11-24T02:47:06,719 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426719 2024-11-24T02:47:06,725 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,725 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,725 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,725 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,725 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,725 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426710 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426719 2024-11-24T02:47:06,726 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,726 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,726 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426719) roll requested 2024-11-24T02:47:06,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426726 2024-11-24T02:47:06,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,734 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,734 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426719 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426726 2024-11-24T02:47:06,735 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,735 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,735 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,735 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426726) roll requested 2024-11-24T02:47:06,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426735 2024-11-24T02:47:06,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,744 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426726 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426735 2024-11-24T02:47:06,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426735) roll requested 2024-11-24T02:47:06,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426745 2024-11-24T02:47:06,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426735 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426745 2024-11-24T02:47:06,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,753 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426745) roll requested 2024-11-24T02:47:06,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426753 2024-11-24T02:47:06,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426745 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426753 2024-11-24T02:47:06,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426753) roll requested 2024-11-24T02:47:06,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426759 2024-11-24T02:47:06,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,765 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426753 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426759 2024-11-24T02:47:06,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,765 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426759) roll requested 2024-11-24T02:47:06,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426765 2024-11-24T02:47:06,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426759 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426765 2024-11-24T02:47:06,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,772 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426765) roll requested 2024-11-24T02:47:06,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426773 2024-11-24T02:47:06,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426765 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426773 2024-11-24T02:47:06,781 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,781 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426773) roll requested 2024-11-24T02:47:06,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426781 2024-11-24T02:47:06,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,788 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,788 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,788 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426773 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426781 2024-11-24T02:47:06,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,789 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426781) roll requested 2024-11-24T02:47:06,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426789 2024-11-24T02:47:06,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426781 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426789 2024-11-24T02:47:06,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,796 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426789) roll requested 2024-11-24T02:47:06,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426796 2024-11-24T02:47:06,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426789 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426796 2024-11-24T02:47:06,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,805 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426796) roll requested 2024-11-24T02:47:06,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426805 2024-11-24T02:47:06,812 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,812 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426796 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426805 2024-11-24T02:47:06,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,813 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426805) roll requested 2024-11-24T02:47:06,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426813 2024-11-24T02:47:06,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426805 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426813 2024-11-24T02:47:06,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,818 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426813) roll requested 2024-11-24T02:47:06,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426818 2024-11-24T02:47:06,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,822 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,822 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,822 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426813 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426818 2024-11-24T02:47:06,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,822 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426818) roll requested 2024-11-24T02:47:06,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426823 2024-11-24T02:47:06,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426818 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426823 2024-11-24T02:47:06,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,829 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426823) roll requested 2024-11-24T02:47:06,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426829 2024-11-24T02:47:06,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426823 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426829 2024-11-24T02:47:06,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,836 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426829) roll requested 2024-11-24T02:47:06,836 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426836 2024-11-24T02:47:06,843 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,843 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,843 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,843 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,843 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,843 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426829 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426836 2024-11-24T02:47:06,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,844 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426836) roll requested 2024-11-24T02:47:06,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426844 2024-11-24T02:47:06,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426836 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426844 2024-11-24T02:47:06,853 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,853 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,853 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,854 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426844) roll requested 2024-11-24T02:47:06,854 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426854 2024-11-24T02:47:06,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,862 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426844 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426854 2024-11-24T02:47:06,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,863 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,863 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426854) roll requested 2024-11-24T02:47:06,863 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426863 2024-11-24T02:47:06,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426854 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426863 2024-11-24T02:47:06,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,872 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426863) roll requested 2024-11-24T02:47:06,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426872 2024-11-24T02:47:06,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426863 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426872 2024-11-24T02:47:06,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,878 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426872) roll requested 2024-11-24T02:47:06,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426878 2024-11-24T02:47:06,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426872 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426878 2024-11-24T02:47:06,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426878) roll requested 2024-11-24T02:47:06,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426890 2024-11-24T02:47:06,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426878 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426890 2024-11-24T02:47:06,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426890) roll requested 2024-11-24T02:47:06,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426900 2024-11-24T02:47:06,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426900 2024-11-24T02:47:06,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,908 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426900) roll requested 2024-11-24T02:47:06,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426908 2024-11-24T02:47:06,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426900 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426908 2024-11-24T02:47:06,914 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,914 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426908) roll requested 2024-11-24T02:47:06,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426914 2024-11-24T02:47:06,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426914 2024-11-24T02:47:06,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,919 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426914) roll requested 2024-11-24T02:47:06,919 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426919 2024-11-24T02:47:06,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,923 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426914 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426919 2024-11-24T02:47:06,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,924 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426919) roll requested 2024-11-24T02:47:06,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426924 2024-11-24T02:47:06,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426919 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426924 2024-11-24T02:47:06,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,931 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426924) roll requested 2024-11-24T02:47:06,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426931 2024-11-24T02:47:06,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426924 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426931 2024-11-24T02:47:06,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,937 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426931) roll requested 2024-11-24T02:47:06,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426937 2024-11-24T02:47:06,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,943 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426931 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426937 2024-11-24T02:47:06,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,945 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426937) roll requested 2024-11-24T02:47:06,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426945 2024-11-24T02:47:06,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,952 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,952 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,952 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426937 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426945 2024-11-24T02:47:06,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,952 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426945) roll requested 2024-11-24T02:47:06,953 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426953 2024-11-24T02:47:06,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426945 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426953 2024-11-24T02:47:06,959 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,959 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426953) roll requested 2024-11-24T02:47:06,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426960 2024-11-24T02:47:06,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426953 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426960 2024-11-24T02:47:06,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,976 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,976 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426960) roll requested 2024-11-24T02:47:06,976 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426976 2024-11-24T02:47:06,981 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,981 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,981 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,981 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,981 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,981 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426960 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426976 2024-11-24T02:47:06,982 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,982 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,982 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426976) roll requested 2024-11-24T02:47:06,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426982 2024-11-24T02:47:06,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426976 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426982 2024-11-24T02:47:06,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,989 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,989 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426982) roll requested 2024-11-24T02:47:06,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426989 2024-11-24T02:47:06,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426982 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426989 2024-11-24T02:47:06,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,994 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426989) roll requested 2024-11-24T02:47:06,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426995 2024-11-24T02:47:06,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:06,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426989 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426995 2024-11-24T02:47:06,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:06,999 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:06,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426995) roll requested 2024-11-24T02:47:06,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416426999 2024-11-24T02:47:07,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426995 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426999 2024-11-24T02:47:07,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,006 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,006 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416426999) roll requested 2024-11-24T02:47:07,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427006 2024-11-24T02:47:07,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416426999 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427006 2024-11-24T02:47:07,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,011 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427006) roll requested 2024-11-24T02:47:07,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427012 2024-11-24T02:47:07,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427006 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427012 2024-11-24T02:47:07,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,021 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427012) roll requested 2024-11-24T02:47:07,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427021 2024-11-24T02:47:07,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,028 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427012 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427021 2024-11-24T02:47:07,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,028 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427021) roll requested 2024-11-24T02:47:07,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427029 2024-11-24T02:47:07,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427021 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427029 2024-11-24T02:47:07,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,035 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,035 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427029) roll requested 2024-11-24T02:47:07,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427035 2024-11-24T02:47:07,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427029 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427035 2024-11-24T02:47:07,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,042 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427035) roll requested 2024-11-24T02:47:07,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427043 2024-11-24T02:47:07,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427035 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427043 2024-11-24T02:47:07,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,051 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427043) roll requested 2024-11-24T02:47:07,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427051 2024-11-24T02:47:07,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427043 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427051 2024-11-24T02:47:07,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,059 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427051) roll requested 2024-11-24T02:47:07,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427059 2024-11-24T02:47:07,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427051 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427059 2024-11-24T02:47:07,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,066 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427059) roll requested 2024-11-24T02:47:07,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427066 2024-11-24T02:47:07,072 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427059 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427066 2024-11-24T02:47:07,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,073 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427066) roll requested 2024-11-24T02:47:07,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427073 2024-11-24T02:47:07,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427066 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427073 2024-11-24T02:47:07,080 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,080 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,080 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,080 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427073) roll requested 2024-11-24T02:47:07,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427080 2024-11-24T02:47:07,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427073 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427080 2024-11-24T02:47:07,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,087 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427080) roll requested 2024-11-24T02:47:07,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427088 2024-11-24T02:47:07,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427080 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427088 2024-11-24T02:47:07,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,092 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,093 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427088) roll requested 2024-11-24T02:47:07,093 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427093 2024-11-24T02:47:07,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427088 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427093 2024-11-24T02:47:07,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,098 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427093) roll requested 2024-11-24T02:47:07,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427098 2024-11-24T02:47:07,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,105 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427093 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427098 2024-11-24T02:47:07,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,105 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427098) roll requested 2024-11-24T02:47:07,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427105 2024-11-24T02:47:07,111 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,111 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,111 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427098 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427105 2024-11-24T02:47:07,112 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,112 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427105) roll requested 2024-11-24T02:47:07,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427112 2024-11-24T02:47:07,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427105 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427112 2024-11-24T02:47:07,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,118 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427112) roll requested 2024-11-24T02:47:07,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427118 2024-11-24T02:47:07,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,125 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427112 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427118 2024-11-24T02:47:07,126 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,126 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,126 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,126 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427118) roll requested 2024-11-24T02:47:07,126 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427126 2024-11-24T02:47:07,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427118 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427126 2024-11-24T02:47:07,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,134 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,134 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427126) roll requested 2024-11-24T02:47:07,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427134 2024-11-24T02:47:07,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427126 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427134 2024-11-24T02:47:07,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,144 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,144 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427134) roll requested 2024-11-24T02:47:07,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427145 2024-11-24T02:47:07,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427134 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427145 2024-11-24T02:47:07,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,153 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427145) roll requested 2024-11-24T02:47:07,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427153 2024-11-24T02:47:07,159 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,159 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427145 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427153 2024-11-24T02:47:07,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,161 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427153) roll requested 2024-11-24T02:47:07,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427161 2024-11-24T02:47:07,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427153 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427161 2024-11-24T02:47:07,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,170 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427161) roll requested 2024-11-24T02:47:07,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427170 2024-11-24T02:47:07,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427161 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427170 2024-11-24T02:47:07,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,179 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427170) roll requested 2024-11-24T02:47:07,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427180 2024-11-24T02:47:07,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427170 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427180 2024-11-24T02:47:07,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,188 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427180) roll requested 2024-11-24T02:47:07,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427188 2024-11-24T02:47:07,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,196 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,196 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,196 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427180 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427188 2024-11-24T02:47:07,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,197 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427188) roll requested 2024-11-24T02:47:07,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427197 2024-11-24T02:47:07,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,203 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,203 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,203 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,203 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427188 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427197 2024-11-24T02:47:07,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,203 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427197) roll requested 2024-11-24T02:47:07,204 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427204 2024-11-24T02:47:07,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427197 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427204 2024-11-24T02:47:07,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,211 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,212 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427204) roll requested 2024-11-24T02:47:07,212 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427212 2024-11-24T02:47:07,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427204 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427212 2024-11-24T02:47:07,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,220 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427212) roll requested 2024-11-24T02:47:07,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427220 2024-11-24T02:47:07,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427212 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427220 2024-11-24T02:47:07,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,229 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427220) roll requested 2024-11-24T02:47:07,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427229 2024-11-24T02:47:07,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427220 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427229 2024-11-24T02:47:07,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,236 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,236 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427229) roll requested 2024-11-24T02:47:07,236 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427236 2024-11-24T02:47:07,245 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,245 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,245 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427229 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427236 2024-11-24T02:47:07,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,246 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,246 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427236) roll requested 2024-11-24T02:47:07,246 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427246 2024-11-24T02:47:07,252 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,252 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,252 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427236 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427246 2024-11-24T02:47:07,253 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,253 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427246) roll requested 2024-11-24T02:47:07,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427253 2024-11-24T02:47:07,258 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,258 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,258 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,259 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427246 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427253 2024-11-24T02:47:07,259 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,259 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427253) roll requested 2024-11-24T02:47:07,260 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427259 2024-11-24T02:47:07,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,267 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,267 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427253 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427259 2024-11-24T02:47:07,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,267 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427259) roll requested 2024-11-24T02:47:07,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427268 2024-11-24T02:47:07,275 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,276 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,276 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,276 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,276 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427259 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427268 2024-11-24T02:47:07,277 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,277 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,277 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427268) roll requested 2024-11-24T02:47:07,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427277 2024-11-24T02:47:07,285 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,285 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,285 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,285 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,286 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,286 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427268 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427277 2024-11-24T02:47:07,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,287 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,287 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427277) roll requested 2024-11-24T02:47:07,287 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427287 2024-11-24T02:47:07,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427277 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427287 2024-11-24T02:47:07,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,301 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427287) roll requested 2024-11-24T02:47:07,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427301 2024-11-24T02:47:07,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427287 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427301 2024-11-24T02:47:07,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,324 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427301) roll requested 2024-11-24T02:47:07,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427325 2024-11-24T02:47:07,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,370 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427301 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427325 2024-11-24T02:47:07,371 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,371 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,371 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,371 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427325) roll requested 2024-11-24T02:47:07,371 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427371 2024-11-24T02:47:07,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427325 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427371 2024-11-24T02:47:07,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,378 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427371) roll requested 2024-11-24T02:47:07,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427379 2024-11-24T02:47:07,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,385 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,385 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,385 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427371 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427379 2024-11-24T02:47:07,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,386 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,386 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427379) roll requested 2024-11-24T02:47:07,386 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427386 2024-11-24T02:47:07,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427379 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427386 2024-11-24T02:47:07,394 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,394 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,394 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,394 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427386) roll requested 2024-11-24T02:47:07,394 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427394 2024-11-24T02:47:07,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427386 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427394 2024-11-24T02:47:07,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,403 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427394) roll requested 2024-11-24T02:47:07,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427403 2024-11-24T02:47:07,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,412 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,412 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,412 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427394 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427403 2024-11-24T02:47:07,413 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,413 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,413 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,413 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427403) roll requested 2024-11-24T02:47:07,414 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427413 2024-11-24T02:47:07,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427403 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427413 2024-11-24T02:47:07,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,421 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427413) roll requested 2024-11-24T02:47:07,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427422 2024-11-24T02:47:07,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,429 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,429 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427413 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427422 2024-11-24T02:47:07,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,430 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427422) roll requested 2024-11-24T02:47:07,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427431 2024-11-24T02:47:07,439 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,439 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,439 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,439 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,439 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427422 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427431 2024-11-24T02:47:07,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,440 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427431) roll requested 2024-11-24T02:47:07,440 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427440 2024-11-24T02:47:07,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427431 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427440 2024-11-24T02:47:07,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,449 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,449 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427440) roll requested 2024-11-24T02:47:07,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427450 2024-11-24T02:47:07,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427440 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427450 2024-11-24T02:47:07,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,457 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427450) roll requested 2024-11-24T02:47:07,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427457 2024-11-24T02:47:07,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427450 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427457 2024-11-24T02:47:07,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,464 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427457) roll requested 2024-11-24T02:47:07,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427464 2024-11-24T02:47:07,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427457 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427464 2024-11-24T02:47:07,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,471 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,471 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427464) roll requested 2024-11-24T02:47:07,471 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427471 2024-11-24T02:47:07,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427464 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427471 2024-11-24T02:47:07,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,479 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427471) roll requested 2024-11-24T02:47:07,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427479 2024-11-24T02:47:07,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427471 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427479 2024-11-24T02:47:07,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,487 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427479) roll requested 2024-11-24T02:47:07,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427487 2024-11-24T02:47:07,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427479 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427487 2024-11-24T02:47:07,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,497 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427487) roll requested 2024-11-24T02:47:07,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427497 2024-11-24T02:47:07,504 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,504 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427487 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427497 2024-11-24T02:47:07,505 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,505 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427497) roll requested 2024-11-24T02:47:07,506 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427505 2024-11-24T02:47:07,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427497 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427505 2024-11-24T02:47:07,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,514 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427505) roll requested 2024-11-24T02:47:07,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427514 2024-11-24T02:47:07,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,521 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,521 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427505 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427514 2024-11-24T02:47:07,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,522 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,522 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427514) roll requested 2024-11-24T02:47:07,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427523 2024-11-24T02:47:07,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427514 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427523 2024-11-24T02:47:07,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,530 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427523) roll requested 2024-11-24T02:47:07,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427530 2024-11-24T02:47:07,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427523 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427530 2024-11-24T02:47:07,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,537 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427530) roll requested 2024-11-24T02:47:07,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427537 2024-11-24T02:47:07,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,545 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427530 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427537 2024-11-24T02:47:07,546 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,546 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,546 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427537) roll requested 2024-11-24T02:47:07,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427546 2024-11-24T02:47:07,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427537 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427546 2024-11-24T02:47:07,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,555 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427546) roll requested 2024-11-24T02:47:07,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427555 2024-11-24T02:47:07,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427546 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427555 2024-11-24T02:47:07,564 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,564 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427555) roll requested 2024-11-24T02:47:07,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427564 2024-11-24T02:47:07,570 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,570 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,570 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,571 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,571 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427555 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427564 2024-11-24T02:47:07,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,571 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,572 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427564) roll requested 2024-11-24T02:47:07,572 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427572 2024-11-24T02:47:07,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427564 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427572 2024-11-24T02:47:07,579 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,579 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427572) roll requested 2024-11-24T02:47:07,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427580 2024-11-24T02:47:07,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427572 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427580 2024-11-24T02:47:07,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,585 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427580) roll requested 2024-11-24T02:47:07,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427586 2024-11-24T02:47:07,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,592 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,592 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427580 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427586 2024-11-24T02:47:07,593 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,593 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427586) roll requested 2024-11-24T02:47:07,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427594 2024-11-24T02:47:07,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,601 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,601 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,601 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427586 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427594 2024-11-24T02:47:07,602 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,602 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427594) roll requested 2024-11-24T02:47:07,603 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427603 2024-11-24T02:47:07,608 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,608 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,608 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,608 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,608 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,608 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427594 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427603 2024-11-24T02:47:07,609 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,609 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427603) roll requested 2024-11-24T02:47:07,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427609 2024-11-24T02:47:07,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,614 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,614 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427603 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427609 2024-11-24T02:47:07,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,616 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,616 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427609) roll requested 2024-11-24T02:47:07,616 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427616 2024-11-24T02:47:07,622 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427609 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427616 2024-11-24T02:47:07,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,624 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427616) roll requested 2024-11-24T02:47:07,624 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427624 2024-11-24T02:47:07,630 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,630 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,630 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,630 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,631 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,631 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427616 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427624 2024-11-24T02:47:07,631 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,631 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,631 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,631 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427624) roll requested 2024-11-24T02:47:07,631 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427631 2024-11-24T02:47:07,636 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,637 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,637 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,637 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,637 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,637 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427624 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427631 2024-11-24T02:47:07,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,637 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427631) roll requested 2024-11-24T02:47:07,638 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427638 2024-11-24T02:47:07,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427631 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427638 2024-11-24T02:47:07,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,645 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427638) roll requested 2024-11-24T02:47:07,646 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427646 2024-11-24T02:47:07,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,653 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,653 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427638 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427646 2024-11-24T02:47:07,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,653 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427646) roll requested 2024-11-24T02:47:07,654 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427654 2024-11-24T02:47:07,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427646 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427654 2024-11-24T02:47:07,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,663 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427654) roll requested 2024-11-24T02:47:07,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427663 2024-11-24T02:47:07,671 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,671 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,671 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,671 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,671 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427654 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427663 2024-11-24T02:47:07,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,672 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427663) roll requested 2024-11-24T02:47:07,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427672 2024-11-24T02:47:07,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427663 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427672 2024-11-24T02:47:07,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,681 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427672) roll requested 2024-11-24T02:47:07,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427681 2024-11-24T02:47:07,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427672 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427681 2024-11-24T02:47:07,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,691 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427681) roll requested 2024-11-24T02:47:07,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427691 2024-11-24T02:47:07,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427681 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427691 2024-11-24T02:47:07,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,699 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427691) roll requested 2024-11-24T02:47:07,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427699 2024-11-24T02:47:07,706 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,706 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,706 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,706 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427691 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427699 2024-11-24T02:47:07,707 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,707 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,707 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427699) roll requested 2024-11-24T02:47:07,708 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427708 2024-11-24T02:47:07,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427699 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427708 2024-11-24T02:47:07,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,716 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427708) roll requested 2024-11-24T02:47:07,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427717 2024-11-24T02:47:07,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427708 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427717 2024-11-24T02:47:07,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,724 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427717) roll requested 2024-11-24T02:47:07,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427725 2024-11-24T02:47:07,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427717 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427725 2024-11-24T02:47:07,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,730 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427725) roll requested 2024-11-24T02:47:07,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427730 2024-11-24T02:47:07,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,736 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,736 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427725 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427730 2024-11-24T02:47:07,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,737 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427730) roll requested 2024-11-24T02:47:07,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427737 2024-11-24T02:47:07,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,744 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427730 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427737 2024-11-24T02:47:07,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,744 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427737) roll requested 2024-11-24T02:47:07,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427744 2024-11-24T02:47:07,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427737 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427744 2024-11-24T02:47:07,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,752 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427744) roll requested 2024-11-24T02:47:07,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427752 2024-11-24T02:47:07,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,759 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,759 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,759 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,759 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,759 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427744 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427752 2024-11-24T02:47:07,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,759 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427752) roll requested 2024-11-24T02:47:07,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427760 2024-11-24T02:47:07,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427752 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427760 2024-11-24T02:47:07,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,767 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427760) roll requested 2024-11-24T02:47:07,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427767 2024-11-24T02:47:07,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427760 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427767 2024-11-24T02:47:07,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,775 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427767) roll requested 2024-11-24T02:47:07,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427776 2024-11-24T02:47:07,784 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427767 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427776 2024-11-24T02:47:07,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,786 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427776) roll requested 2024-11-24T02:47:07,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427786 2024-11-24T02:47:07,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427776 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427786 2024-11-24T02:47:07,801 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,801 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,801 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,801 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427786) roll requested 2024-11-24T02:47:07,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427801 2024-11-24T02:47:07,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427786 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427801 2024-11-24T02:47:07,806 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,806 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,806 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427801) roll requested 2024-11-24T02:47:07,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427807 2024-11-24T02:47:07,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427801 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427807 2024-11-24T02:47:07,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,811 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427807) roll requested 2024-11-24T02:47:07,812 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427811 2024-11-24T02:47:07,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427807 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427811 2024-11-24T02:47:07,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,816 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427811) roll requested 2024-11-24T02:47:07,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427817 2024-11-24T02:47:07,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427811 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427817 2024-11-24T02:47:07,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,824 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427817) roll requested 2024-11-24T02:47:07,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427824 2024-11-24T02:47:07,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427817 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427824 2024-11-24T02:47:07,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,831 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427824) roll requested 2024-11-24T02:47:07,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427832 2024-11-24T02:47:07,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,838 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427824 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427832 2024-11-24T02:47:07,839 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,839 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427832) roll requested 2024-11-24T02:47:07,840 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427840 2024-11-24T02:47:07,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427832 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427840 2024-11-24T02:47:07,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,849 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427840) roll requested 2024-11-24T02:47:07,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427849 2024-11-24T02:47:07,854 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,854 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427840 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427849 2024-11-24T02:47:07,855 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,855 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,855 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,855 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427849) roll requested 2024-11-24T02:47:07,855 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427855 2024-11-24T02:47:07,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427849 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427855 2024-11-24T02:47:07,862 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,862 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,862 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427855) roll requested 2024-11-24T02:47:07,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427862 2024-11-24T02:47:07,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,870 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,870 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,870 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427855 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427862 2024-11-24T02:47:07,871 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,871 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427862) roll requested 2024-11-24T02:47:07,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427871 2024-11-24T02:47:07,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427862 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427871 2024-11-24T02:47:07,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,881 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427871) roll requested 2024-11-24T02:47:07,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427881 2024-11-24T02:47:07,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,889 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427871 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427881 2024-11-24T02:47:07,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,890 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427881) roll requested 2024-11-24T02:47:07,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427890 2024-11-24T02:47:07,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427881 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427890 2024-11-24T02:47:07,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,899 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427890) roll requested 2024-11-24T02:47:07,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427899 2024-11-24T02:47:07,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427890 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427899 2024-11-24T02:47:07,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,907 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427899) roll requested 2024-11-24T02:47:07,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427908 2024-11-24T02:47:07,916 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427899 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427908 2024-11-24T02:47:07,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,917 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427908) roll requested 2024-11-24T02:47:07,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427918 2024-11-24T02:47:07,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,927 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,927 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,927 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,927 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427908 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427918 2024-11-24T02:47:07,928 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,928 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,928 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427918) roll requested 2024-11-24T02:47:07,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427928 2024-11-24T02:47:07,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,934 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,934 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,934 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,934 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,934 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427918 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427928 2024-11-24T02:47:07,934 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,934 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,935 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,935 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427928) roll requested 2024-11-24T02:47:07,935 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427935 2024-11-24T02:47:07,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,939 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,939 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,939 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,939 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,939 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427928 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427935 2024-11-24T02:47:07,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,939 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427935) roll requested 2024-11-24T02:47:07,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427939 2024-11-24T02:47:07,943 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,943 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,943 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,944 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,944 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,944 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427935 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427939 2024-11-24T02:47:07,944 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,944 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,944 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,944 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427939) roll requested 2024-11-24T02:47:07,945 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427944 2024-11-24T02:47:07,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427939 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427944 2024-11-24T02:47:07,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,951 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427944) roll requested 2024-11-24T02:47:07,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427951 2024-11-24T02:47:07,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427944 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427951 2024-11-24T02:47:07,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,958 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427951) roll requested 2024-11-24T02:47:07,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427958 2024-11-24T02:47:07,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,964 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,964 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,964 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,964 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427951 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427958 2024-11-24T02:47:07,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,964 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427958) roll requested 2024-11-24T02:47:07,965 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427965 2024-11-24T02:47:07,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427958 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427965 2024-11-24T02:47:07,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,971 ERROR [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.append(TestFailedAppendAndSync.java:166) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:268) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.doAppend(FSHLog.java:51) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendEntry(AbstractFSWAL.java:1333) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.appendAndSync(AbstractFSWAL.java:1724) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.consume(AbstractFSWAL.java:1832) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:07,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427965) roll requested 2024-11-24T02:47:07,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427971 2024-11-24T02:47:07,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427965 with entries=0, filesize=85 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427971 2024-11-24T02:47:07,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:07,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:07,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427971) roll requested 2024-11-24T02:47:07,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427978 2024-11-24T02:47:07,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427978 2024-11-24T02:47:07,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:07,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:07,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427978) roll requested 2024-11-24T02:47:07,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427983 2024-11-24T02:47:07,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427983 2024-11-24T02:47:07,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,990 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:07,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:07,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427983) roll requested 2024-11-24T02:47:07,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427990 2024-11-24T02:47:07,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,997 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,997 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,997 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,997 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:07,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427990 2024-11-24T02:47:07,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:07,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:07,998 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:07,998 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427990) roll requested 2024-11-24T02:47:07,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416427998 2024-11-24T02:47:08,005 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,005 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,005 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,005 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427998 2024-11-24T02:47:08,006 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416427998) roll requested 2024-11-24T02:47:08,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428007 2024-11-24T02:47:08,014 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,014 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,015 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,015 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,015 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,015 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416427998 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428007 2024-11-24T02:47:08,016 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,016 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,016 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428007) roll requested 2024-11-24T02:47:08,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428017 2024-11-24T02:47:08,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428017 2024-11-24T02:47:08,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,029 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,029 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,029 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428017) roll requested 2024-11-24T02:47:08,029 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428029 2024-11-24T02:47:08,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,036 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,036 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,036 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428029 2024-11-24T02:47:08,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,037 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428029) roll requested 2024-11-24T02:47:08,037 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428037 2024-11-24T02:47:08,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,042 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,042 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,042 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,042 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428029 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428037 2024-11-24T02:47:08,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428037) roll requested 2024-11-24T02:47:08,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428043 2024-11-24T02:47:08,048 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,048 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,048 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428037 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428043 2024-11-24T02:47:08,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428043) roll requested 2024-11-24T02:47:08,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428050 2024-11-24T02:47:08,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428050 2024-11-24T02:47:08,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428050) roll requested 2024-11-24T02:47:08,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428061 2024-11-24T02:47:08,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428061 2024-11-24T02:47:08,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428061) roll requested 2024-11-24T02:47:08,067 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428067 2024-11-24T02:47:08,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,075 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428067 2024-11-24T02:47:08,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428067) roll requested 2024-11-24T02:47:08,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428075 2024-11-24T02:47:08,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428067 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428075 2024-11-24T02:47:08,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428075) roll requested 2024-11-24T02:47:08,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428082 2024-11-24T02:47:08,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,086 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,086 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428082 2024-11-24T02:47:08,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428082) roll requested 2024-11-24T02:47:08,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428087 2024-11-24T02:47:08,091 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,091 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,091 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,091 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,091 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,091 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428087 2024-11-24T02:47:08,092 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,092 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,092 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,092 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428087) roll requested 2024-11-24T02:47:08,092 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428092 2024-11-24T02:47:08,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428092 2024-11-24T02:47:08,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428092) roll requested 2024-11-24T02:47:08,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428098 2024-11-24T02:47:08,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428092 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428098 2024-11-24T02:47:08,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428098) roll requested 2024-11-24T02:47:08,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428104 2024-11-24T02:47:08,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428104 2024-11-24T02:47:08,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,111 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,111 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428104) roll requested 2024-11-24T02:47:08,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428112 2024-11-24T02:47:08,118 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,118 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,118 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,119 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,119 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,119 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428112 2024-11-24T02:47:08,119 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,120 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,120 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428112) roll requested 2024-11-24T02:47:08,120 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428120 2024-11-24T02:47:08,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428120 2024-11-24T02:47:08,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,130 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428120) roll requested 2024-11-24T02:47:08,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428130 2024-11-24T02:47:08,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428120 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428130 2024-11-24T02:47:08,138 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,138 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,138 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428130) roll requested 2024-11-24T02:47:08,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428138 2024-11-24T02:47:08,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,145 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428138 2024-11-24T02:47:08,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,147 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428138) roll requested 2024-11-24T02:47:08,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428147 2024-11-24T02:47:08,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428147 2024-11-24T02:47:08,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428147) roll requested 2024-11-24T02:47:08,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428155 2024-11-24T02:47:08,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428155 2024-11-24T02:47:08,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428155) roll requested 2024-11-24T02:47:08,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428162 2024-11-24T02:47:08,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428162 2024-11-24T02:47:08,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428162) roll requested 2024-11-24T02:47:08,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428171 2024-11-24T02:47:08,178 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,178 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,178 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,178 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,178 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,178 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428171 2024-11-24T02:47:08,179 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,179 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,179 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,179 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428171) roll requested 2024-11-24T02:47:08,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428180 2024-11-24T02:47:08,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,189 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,189 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428180 2024-11-24T02:47:08,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,190 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428180) roll requested 2024-11-24T02:47:08,190 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428190 2024-11-24T02:47:08,199 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,199 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,199 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,199 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,199 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428190 2024-11-24T02:47:08,200 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,200 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,200 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428190) roll requested 2024-11-24T02:47:08,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428200 2024-11-24T02:47:08,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428190 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428200 2024-11-24T02:47:08,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,206 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428200) roll requested 2024-11-24T02:47:08,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428206 2024-11-24T02:47:08,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428206 2024-11-24T02:47:08,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428206) roll requested 2024-11-24T02:47:08,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428211 2024-11-24T02:47:08,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428206 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428211 2024-11-24T02:47:08,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,216 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428211) roll requested 2024-11-24T02:47:08,216 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428216 2024-11-24T02:47:08,222 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,222 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428216 2024-11-24T02:47:08,223 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428216) roll requested 2024-11-24T02:47:08,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428224 2024-11-24T02:47:08,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428216 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428224 2024-11-24T02:47:08,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428224) roll requested 2024-11-24T02:47:08,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428229 2024-11-24T02:47:08,234 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,234 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,234 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,234 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,234 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,234 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428229 2024-11-24T02:47:08,235 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,235 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,235 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,235 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428229) roll requested 2024-11-24T02:47:08,235 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428235 2024-11-24T02:47:08,241 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,241 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,241 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,241 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,241 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,241 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428235 2024-11-24T02:47:08,242 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,242 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,242 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,242 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428235) roll requested 2024-11-24T02:47:08,242 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428242 2024-11-24T02:47:08,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,248 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428235 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428242 2024-11-24T02:47:08,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,249 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428242) roll requested 2024-11-24T02:47:08,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428250 2024-11-24T02:47:08,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428242 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428250 2024-11-24T02:47:08,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428250) roll requested 2024-11-24T02:47:08,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428258 2024-11-24T02:47:08,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,265 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428258 2024-11-24T02:47:08,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428258) roll requested 2024-11-24T02:47:08,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428267 2024-11-24T02:47:08,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,275 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428267 2024-11-24T02:47:08,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428267) roll requested 2024-11-24T02:47:08,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428277 2024-11-24T02:47:08,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428277 2024-11-24T02:47:08,285 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,285 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428277) roll requested 2024-11-24T02:47:08,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428285 2024-11-24T02:47:08,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428285 2024-11-24T02:47:08,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428285) roll requested 2024-11-24T02:47:08,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428293 2024-11-24T02:47:08,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428293 2024-11-24T02:47:08,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428293) roll requested 2024-11-24T02:47:08,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428300 2024-11-24T02:47:08,306 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,306 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,306 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,307 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,307 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428300 2024-11-24T02:47:08,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,308 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,308 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,308 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428300) roll requested 2024-11-24T02:47:08,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428308 2024-11-24T02:47:08,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428308 2024-11-24T02:47:08,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428308) roll requested 2024-11-24T02:47:08,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428316 2024-11-24T02:47:08,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428316 2024-11-24T02:47:08,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,326 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,326 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428316) roll requested 2024-11-24T02:47:08,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428326 2024-11-24T02:47:08,334 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,334 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428326 2024-11-24T02:47:08,335 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428326) roll requested 2024-11-24T02:47:08,336 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428336 2024-11-24T02:47:08,344 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428326 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428336 2024-11-24T02:47:08,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,346 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428336) roll requested 2024-11-24T02:47:08,346 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428346 2024-11-24T02:47:08,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,351 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428346 2024-11-24T02:47:08,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428346) roll requested 2024-11-24T02:47:08,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428353 2024-11-24T02:47:08,358 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428346 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428353 2024-11-24T02:47:08,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428353) roll requested 2024-11-24T02:47:08,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428360 2024-11-24T02:47:08,367 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,367 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,368 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428360 2024-11-24T02:47:08,369 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,369 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,369 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,369 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428360) roll requested 2024-11-24T02:47:08,370 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428370 2024-11-24T02:47:08,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,377 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,377 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428370 2024-11-24T02:47:08,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,379 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428370) roll requested 2024-11-24T02:47:08,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428379 2024-11-24T02:47:08,387 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,387 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428370 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428379 2024-11-24T02:47:08,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428379) roll requested 2024-11-24T02:47:08,390 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428390 2024-11-24T02:47:08,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,396 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,396 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428379 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428390 2024-11-24T02:47:08,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,397 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428390) roll requested 2024-11-24T02:47:08,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428397 2024-11-24T02:47:08,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428397 2024-11-24T02:47:08,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428397) roll requested 2024-11-24T02:47:08,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428403 2024-11-24T02:47:08,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428403 2024-11-24T02:47:08,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=56, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428403) roll requested 2024-11-24T02:47:08,413 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428412 2024-11-24T02:47:08,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,420 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428412 2024-11-24T02:47:08,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=57, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,421 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,421 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,421 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428412) roll requested 2024-11-24T02:47:08,421 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428421 2024-11-24T02:47:08,426 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428421 2024-11-24T02:47:08,427 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,427 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=58, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428421) roll requested 2024-11-24T02:47:08,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428428 2024-11-24T02:47:08,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428421 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428428 2024-11-24T02:47:08,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,435 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=59, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,435 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,435 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428428) roll requested 2024-11-24T02:47:08,435 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428435 2024-11-24T02:47:08,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428435 2024-11-24T02:47:08,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=60, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428435) roll requested 2024-11-24T02:47:08,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428443 2024-11-24T02:47:08,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428435 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428443 2024-11-24T02:47:08,450 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=61, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,450 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,450 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428443) roll requested 2024-11-24T02:47:08,451 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428450 2024-11-24T02:47:08,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428450 2024-11-24T02:47:08,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=62, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428450) roll requested 2024-11-24T02:47:08,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428459 2024-11-24T02:47:08,466 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,466 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,466 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,466 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428459 2024-11-24T02:47:08,467 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=63, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,468 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428459) roll requested 2024-11-24T02:47:08,468 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428468 2024-11-24T02:47:08,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428468 2024-11-24T02:47:08,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=64, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428468) roll requested 2024-11-24T02:47:08,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428479 2024-11-24T02:47:08,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428468 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428479 2024-11-24T02:47:08,486 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=65, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428479) roll requested 2024-11-24T02:47:08,487 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428487 2024-11-24T02:47:08,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428487 2024-11-24T02:47:08,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=66, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,492 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428487) roll requested 2024-11-24T02:47:08,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428492 2024-11-24T02:47:08,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,497 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,497 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,498 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428492 2024-11-24T02:47:08,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=67, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428492) roll requested 2024-11-24T02:47:08,499 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428499 2024-11-24T02:47:08,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,504 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,504 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428499 2024-11-24T02:47:08,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=68, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428499) roll requested 2024-11-24T02:47:08,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428505 2024-11-24T02:47:08,510 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,510 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,510 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,510 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,510 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,510 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428499 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428505 2024-11-24T02:47:08,511 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,511 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=69, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,511 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,511 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,511 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,511 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428505) roll requested 2024-11-24T02:47:08,511 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428511 2024-11-24T02:47:08,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428511 2024-11-24T02:47:08,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=70, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428511) roll requested 2024-11-24T02:47:08,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428521 2024-11-24T02:47:08,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428511 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428521 2024-11-24T02:47:08,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=71, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428521) roll requested 2024-11-24T02:47:08,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428531 2024-11-24T02:47:08,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428531 2024-11-24T02:47:08,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=72, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428531) roll requested 2024-11-24T02:47:08,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428540 2024-11-24T02:47:08,547 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,547 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,547 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,547 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,547 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,547 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428540 2024-11-24T02:47:08,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=73, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,548 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,548 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428540) roll requested 2024-11-24T02:47:08,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428548 2024-11-24T02:47:08,555 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,555 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,555 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,556 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,556 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428548 2024-11-24T02:47:08,556 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,556 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=74, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,556 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,556 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,556 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428548) roll requested 2024-11-24T02:47:08,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428556 2024-11-24T02:47:08,561 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,561 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,561 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,561 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,561 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,561 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428556 2024-11-24T02:47:08,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,562 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=75, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,562 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,562 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,562 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,562 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428556) roll requested 2024-11-24T02:47:08,562 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428562 2024-11-24T02:47:08,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428562 2024-11-24T02:47:08,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=76, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428562) roll requested 2024-11-24T02:47:08,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428568 2024-11-24T02:47:08,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428562 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428568 2024-11-24T02:47:08,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=77, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428568) roll requested 2024-11-24T02:47:08,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428575 2024-11-24T02:47:08,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,582 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428575 2024-11-24T02:47:08,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=78, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,583 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428575) roll requested 2024-11-24T02:47:08,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428583 2024-11-24T02:47:08,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428583 2024-11-24T02:47:08,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=79, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428583) roll requested 2024-11-24T02:47:08,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428592 2024-11-24T02:47:08,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428592 2024-11-24T02:47:08,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,600 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=80, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,600 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428592) roll requested 2024-11-24T02:47:08,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428601 2024-11-24T02:47:08,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,607 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,607 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,607 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428601 2024-11-24T02:47:08,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=81, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428601) roll requested 2024-11-24T02:47:08,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428608 2024-11-24T02:47:08,616 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,616 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,616 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,616 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,616 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,616 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428608 2024-11-24T02:47:08,617 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,617 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=82, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,617 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,617 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428608) roll requested 2024-11-24T02:47:08,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428617 2024-11-24T02:47:08,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428617 2024-11-24T02:47:08,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,626 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=83, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,626 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,626 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428617) roll requested 2024-11-24T02:47:08,626 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428626 2024-11-24T02:47:08,633 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428626 2024-11-24T02:47:08,634 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=84, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428626) roll requested 2024-11-24T02:47:08,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428634 2024-11-24T02:47:08,641 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428626 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428634 2024-11-24T02:47:08,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,642 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=85, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428634) roll requested 2024-11-24T02:47:08,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428643 2024-11-24T02:47:08,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,649 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428643 2024-11-24T02:47:08,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=86, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,650 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,650 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428643) roll requested 2024-11-24T02:47:08,650 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428650 2024-11-24T02:47:08,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,656 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,656 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,656 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428650 2024-11-24T02:47:08,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=87, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,657 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,657 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,657 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,657 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428650) roll requested 2024-11-24T02:47:08,657 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428657 2024-11-24T02:47:08,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428650 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428657 2024-11-24T02:47:08,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=88, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428657) roll requested 2024-11-24T02:47:08,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428664 2024-11-24T02:47:08,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,670 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428657 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428664 2024-11-24T02:47:08,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,671 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=89, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,671 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,671 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,671 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428664) roll requested 2024-11-24T02:47:08,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428671 2024-11-24T02:47:08,678 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,678 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,678 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,678 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,678 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,678 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428671 2024-11-24T02:47:08,679 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,679 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=90, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,679 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428671) roll requested 2024-11-24T02:47:08,680 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428680 2024-11-24T02:47:08,688 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,688 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,688 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,688 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,688 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,688 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428671 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428680 2024-11-24T02:47:08,689 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,689 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=91, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,689 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,689 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,689 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,689 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428680) roll requested 2024-11-24T02:47:08,689 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428689 2024-11-24T02:47:08,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428680 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428689 2024-11-24T02:47:08,697 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=92, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428689) roll requested 2024-11-24T02:47:08,698 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428697 2024-11-24T02:47:08,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,705 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428689 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428697 2024-11-24T02:47:08,706 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,706 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=93, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,706 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,706 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,706 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,706 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428697) roll requested 2024-11-24T02:47:08,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428706 2024-11-24T02:47:08,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,714 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,714 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,714 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428706 2024-11-24T02:47:08,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,715 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=94, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,715 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,715 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,715 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,715 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428706) roll requested 2024-11-24T02:47:08,715 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428715 2024-11-24T02:47:08,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428715 2024-11-24T02:47:08,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=95, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428715) roll requested 2024-11-24T02:47:08,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428725 2024-11-24T02:47:08,733 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,733 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428715 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428725 2024-11-24T02:47:08,734 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=96, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428725) roll requested 2024-11-24T02:47:08,735 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428734 2024-11-24T02:47:08,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428734 2024-11-24T02:47:08,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=97, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428734) roll requested 2024-11-24T02:47:08,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428745 2024-11-24T02:47:08,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,752 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,752 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,752 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428745 2024-11-24T02:47:08,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=98, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428745) roll requested 2024-11-24T02:47:08,753 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428753 2024-11-24T02:47:08,760 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,760 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,760 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,760 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,760 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,760 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428753 2024-11-24T02:47:08,761 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,761 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=99, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,761 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,761 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,761 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,761 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428753) roll requested 2024-11-24T02:47:08,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428761 2024-11-24T02:47:08,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428753 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428761 2024-11-24T02:47:08,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=100, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428761) roll requested 2024-11-24T02:47:08,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428767 2024-11-24T02:47:08,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428767 2024-11-24T02:47:08,772 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=101, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428767) roll requested 2024-11-24T02:47:08,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428773 2024-11-24T02:47:08,779 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,779 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,779 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,779 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,779 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,779 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428773 2024-11-24T02:47:08,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,780 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=102, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,780 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,780 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,780 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428773) roll requested 2024-11-24T02:47:08,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428781 2024-11-24T02:47:08,788 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,788 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,788 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428781 2024-11-24T02:47:08,789 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=103, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,789 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428781) roll requested 2024-11-24T02:47:08,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428789 2024-11-24T02:47:08,794 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,794 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428789 2024-11-24T02:47:08,795 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,795 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=104, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,795 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,795 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428789) roll requested 2024-11-24T02:47:08,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428795 2024-11-24T02:47:08,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,800 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428795 2024-11-24T02:47:08,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=105, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428795) roll requested 2024-11-24T02:47:08,804 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428804 2024-11-24T02:47:08,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,809 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428804 2024-11-24T02:47:08,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,810 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=106, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,810 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,810 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,810 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428804) roll requested 2024-11-24T02:47:08,810 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428810 2024-11-24T02:47:08,814 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,814 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428810 2024-11-24T02:47:08,815 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,815 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=107, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,815 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,815 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,816 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428810) roll requested 2024-11-24T02:47:08,816 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428816 2024-11-24T02:47:08,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428810 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428816 2024-11-24T02:47:08,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=108, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428816) roll requested 2024-11-24T02:47:08,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428824 2024-11-24T02:47:08,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428816 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428824 2024-11-24T02:47:08,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=109, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428824) roll requested 2024-11-24T02:47:08,830 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428830 2024-11-24T02:47:08,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428830 2024-11-24T02:47:08,835 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=110, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,835 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428830) roll requested 2024-11-24T02:47:08,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428835 2024-11-24T02:47:08,839 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,839 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428830 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428835 2024-11-24T02:47:08,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,841 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=111, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,841 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,841 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,841 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428835) roll requested 2024-11-24T02:47:08,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428841 2024-11-24T02:47:08,846 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,846 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428841 2024-11-24T02:47:08,846 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,846 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=112, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,846 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428841) roll requested 2024-11-24T02:47:08,847 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428847 2024-11-24T02:47:08,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,852 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,852 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428841 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428847 2024-11-24T02:47:08,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=113, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428847) roll requested 2024-11-24T02:47:08,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428853 2024-11-24T02:47:08,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428853 2024-11-24T02:47:08,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=114, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428853) roll requested 2024-11-24T02:47:08,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428860 2024-11-24T02:47:08,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428860 2024-11-24T02:47:08,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=115, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428860) roll requested 2024-11-24T02:47:08,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428866 2024-11-24T02:47:08,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428866 2024-11-24T02:47:08,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,872 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=116, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,872 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,872 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428866) roll requested 2024-11-24T02:47:08,872 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428872 2024-11-24T02:47:08,877 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,877 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,877 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,877 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,877 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,877 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428872 2024-11-24T02:47:08,878 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,878 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=117, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,878 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,878 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,878 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,878 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428872) roll requested 2024-11-24T02:47:08,878 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428878 2024-11-24T02:47:08,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,882 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,883 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428872 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428878 2024-11-24T02:47:08,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,883 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=118, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,883 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,883 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,883 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428878) roll requested 2024-11-24T02:47:08,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428883 2024-11-24T02:47:08,887 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,887 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428878 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428883 2024-11-24T02:47:08,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=119, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428883) roll requested 2024-11-24T02:47:08,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428889 2024-11-24T02:47:08,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428889 2024-11-24T02:47:08,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,896 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=120, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428889) roll requested 2024-11-24T02:47:08,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428896 2024-11-24T02:47:08,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,902 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428896 2024-11-24T02:47:08,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=121, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428896) roll requested 2024-11-24T02:47:08,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428904 2024-11-24T02:47:08,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428904 2024-11-24T02:47:08,910 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=122, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428904) roll requested 2024-11-24T02:47:08,911 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428911 2024-11-24T02:47:08,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,915 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,915 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,915 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428911 2024-11-24T02:47:08,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=123, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428911) roll requested 2024-11-24T02:47:08,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428916 2024-11-24T02:47:08,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428916 2024-11-24T02:47:08,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,921 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=124, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,921 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,921 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,921 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428916) roll requested 2024-11-24T02:47:08,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428921 2024-11-24T02:47:08,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428921 2024-11-24T02:47:08,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=125, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428921) roll requested 2024-11-24T02:47:08,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428927 2024-11-24T02:47:08,932 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,932 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,933 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,933 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,933 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,933 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428921 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428927 2024-11-24T02:47:08,933 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,933 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=126, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,934 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,934 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,934 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,934 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428927) roll requested 2024-11-24T02:47:08,934 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428934 2024-11-24T02:47:08,939 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428927 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428934 2024-11-24T02:47:08,940 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,941 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=127, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,941 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,941 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,941 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,941 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428934) roll requested 2024-11-24T02:47:08,941 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428941 2024-11-24T02:47:08,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428934 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428941 2024-11-24T02:47:08,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=128, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428941) roll requested 2024-11-24T02:47:08,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428949 2024-11-24T02:47:08,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428941 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428949 2024-11-24T02:47:08,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=129, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428949) roll requested 2024-11-24T02:47:08,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428961 2024-11-24T02:47:08,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428961 2024-11-24T02:47:08,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=130, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428961) roll requested 2024-11-24T02:47:08,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428967 2024-11-24T02:47:08,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,972 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428967 2024-11-24T02:47:08,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=131, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428967) roll requested 2024-11-24T02:47:08,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428972 2024-11-24T02:47:08,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428972 2024-11-24T02:47:08,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=132, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428972) roll requested 2024-11-24T02:47:08,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428980 2024-11-24T02:47:08,986 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,986 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,987 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,987 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,987 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428980 2024-11-24T02:47:08,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:08,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=133, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:08,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:08,988 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:08,988 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428980) roll requested 2024-11-24T02:47:08,988 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416428988 2024-11-24T02:47:08,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:08,997 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428988 2024-11-24T02:47:09,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,001 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=134, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,001 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,001 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,001 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416428988) roll requested 2024-11-24T02:47:09,001 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429001 2024-11-24T02:47:09,006 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,006 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,006 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,006 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,006 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,006 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416428988 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429001 2024-11-24T02:47:09,007 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,007 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=135, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,007 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,007 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,007 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,007 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429001) roll requested 2024-11-24T02:47:09,007 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429007 2024-11-24T02:47:09,012 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,012 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,012 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,012 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429001 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429007 2024-11-24T02:47:09,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=136, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429007) roll requested 2024-11-24T02:47:09,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429013 2024-11-24T02:47:09,018 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,018 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,018 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,018 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429007 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429013 2024-11-24T02:47:09,019 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,019 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=137, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,019 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,019 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,019 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,019 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429013) roll requested 2024-11-24T02:47:09,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429019 2024-11-24T02:47:09,025 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,025 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,025 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429019 2024-11-24T02:47:09,026 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=138, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429019) roll requested 2024-11-24T02:47:09,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429026 2024-11-24T02:47:09,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429026 2024-11-24T02:47:09,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=139, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,034 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429026) roll requested 2024-11-24T02:47:09,034 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429034 2024-11-24T02:47:09,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429034 2024-11-24T02:47:09,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,042 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=140, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,043 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429034) roll requested 2024-11-24T02:47:09,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429043 2024-11-24T02:47:09,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429034 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429043 2024-11-24T02:47:09,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=141, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,051 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,051 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429043) roll requested 2024-11-24T02:47:09,051 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429051 2024-11-24T02:47:09,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,057 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,057 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429051 2024-11-24T02:47:09,058 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,058 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=142, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,058 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,058 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,058 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,058 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429051) roll requested 2024-11-24T02:47:09,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429059 2024-11-24T02:47:09,065 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,065 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,065 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,065 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,065 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,065 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429051 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429059 2024-11-24T02:47:09,066 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=143, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,066 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429059) roll requested 2024-11-24T02:47:09,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429066 2024-11-24T02:47:09,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429066 2024-11-24T02:47:09,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=144, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429066) roll requested 2024-11-24T02:47:09,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429075 2024-11-24T02:47:09,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429075 2024-11-24T02:47:09,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=145, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429075) roll requested 2024-11-24T02:47:09,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429085 2024-11-24T02:47:09,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429085 2024-11-24T02:47:09,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=146, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,095 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429085) roll requested 2024-11-24T02:47:09,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429095 2024-11-24T02:47:09,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429095 2024-11-24T02:47:09,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=147, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429095) roll requested 2024-11-24T02:47:09,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429106 2024-11-24T02:47:09,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,115 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429106 2024-11-24T02:47:09,115 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,116 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=148, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,116 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,116 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429106) roll requested 2024-11-24T02:47:09,116 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429116 2024-11-24T02:47:09,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429116 2024-11-24T02:47:09,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=149, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429116) roll requested 2024-11-24T02:47:09,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429124 2024-11-24T02:47:09,130 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,131 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,131 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,131 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,131 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,131 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429116 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429124 2024-11-24T02:47:09,131 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,132 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=150, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,132 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,132 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,132 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,132 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429124) roll requested 2024-11-24T02:47:09,132 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429132 2024-11-24T02:47:09,138 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,138 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,138 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429124 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429132 2024-11-24T02:47:09,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=151, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429132) roll requested 2024-11-24T02:47:09,140 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429139 2024-11-24T02:47:09,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429132 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429139 2024-11-24T02:47:09,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=152, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429139) roll requested 2024-11-24T02:47:09,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429148 2024-11-24T02:47:09,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,154 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429148 2024-11-24T02:47:09,155 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,155 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=153, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,155 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,155 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,155 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,155 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429148) roll requested 2024-11-24T02:47:09,155 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429155 2024-11-24T02:47:09,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,161 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,161 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429155 2024-11-24T02:47:09,162 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,162 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=154, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,162 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,162 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,162 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,162 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429155) roll requested 2024-11-24T02:47:09,162 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429162 2024-11-24T02:47:09,169 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,169 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,169 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,169 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,169 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,169 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429155 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429162 2024-11-24T02:47:09,170 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,170 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=155, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,170 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,170 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,170 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,170 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429162) roll requested 2024-11-24T02:47:09,171 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429171 2024-11-24T02:47:09,176 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,176 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429162 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429171 2024-11-24T02:47:09,178 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=156, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429171) roll requested 2024-11-24T02:47:09,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429178 2024-11-24T02:47:09,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,185 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429171 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429178 2024-11-24T02:47:09,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=157, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,186 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,186 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429178) roll requested 2024-11-24T02:47:09,186 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429186 2024-11-24T02:47:09,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,192 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,192 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,192 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,192 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,192 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429186 2024-11-24T02:47:09,196 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,196 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=158, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,196 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,196 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429186) roll requested 2024-11-24T02:47:09,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429197 2024-11-24T02:47:09,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429186 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429197 2024-11-24T02:47:09,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=159, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429197) roll requested 2024-11-24T02:47:09,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429203 2024-11-24T02:47:09,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429203 2024-11-24T02:47:09,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=160, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429203) roll requested 2024-11-24T02:47:09,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429213 2024-11-24T02:47:09,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,218 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429213 2024-11-24T02:47:09,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,220 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=161, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,220 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,220 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429213) roll requested 2024-11-24T02:47:09,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429221 2024-11-24T02:47:09,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429221 2024-11-24T02:47:09,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,230 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=162, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,230 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,230 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,230 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429221) roll requested 2024-11-24T02:47:09,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429230 2024-11-24T02:47:09,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429230 2024-11-24T02:47:09,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,236 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=163, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,236 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429230) roll requested 2024-11-24T02:47:09,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429237 2024-11-24T02:47:09,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,242 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,242 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,242 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,242 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429230 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429237 2024-11-24T02:47:09,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=164, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429237) roll requested 2024-11-24T02:47:09,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429244 2024-11-24T02:47:09,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429244 2024-11-24T02:47:09,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=165, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429244) roll requested 2024-11-24T02:47:09,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429253 2024-11-24T02:47:09,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429253 2024-11-24T02:47:09,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=166, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429253) roll requested 2024-11-24T02:47:09,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429258 2024-11-24T02:47:09,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429258 2024-11-24T02:47:09,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=167, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429258) roll requested 2024-11-24T02:47:09,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429264 2024-11-24T02:47:09,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,269 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,269 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,269 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429264 2024-11-24T02:47:09,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=168, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429264) roll requested 2024-11-24T02:47:09,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429272 2024-11-24T02:47:09,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,278 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,278 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,278 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,278 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,278 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429272 2024-11-24T02:47:09,278 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,278 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=169, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,278 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,279 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,279 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429272) roll requested 2024-11-24T02:47:09,279 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429279 2024-11-24T02:47:09,283 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,283 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429279 2024-11-24T02:47:09,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=170, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429279) roll requested 2024-11-24T02:47:09,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429285 2024-11-24T02:47:09,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429279 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429285 2024-11-24T02:47:09,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,290 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=171, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,290 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,290 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429285) roll requested 2024-11-24T02:47:09,290 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429290 2024-11-24T02:47:09,294 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,294 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,294 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,295 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,295 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,295 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429290 2024-11-24T02:47:09,295 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,295 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=172, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,295 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,295 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,295 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,295 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429290) roll requested 2024-11-24T02:47:09,296 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429295 2024-11-24T02:47:09,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429290 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429295 2024-11-24T02:47:09,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=173, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429295) roll requested 2024-11-24T02:47:09,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429301 2024-11-24T02:47:09,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429295 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429301 2024-11-24T02:47:09,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=174, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429301) roll requested 2024-11-24T02:47:09,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429306 2024-11-24T02:47:09,311 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429306 2024-11-24T02:47:09,312 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=175, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,312 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429306) roll requested 2024-11-24T02:47:09,313 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429312 2024-11-24T02:47:09,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,317 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429312 2024-11-24T02:47:09,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=176, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429312) roll requested 2024-11-24T02:47:09,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429319 2024-11-24T02:47:09,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,324 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429319 2024-11-24T02:47:09,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=177, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429319) roll requested 2024-11-24T02:47:09,325 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429325 2024-11-24T02:47:09,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429325 2024-11-24T02:47:09,330 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,330 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=178, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,330 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,331 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,331 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,331 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429325) roll requested 2024-11-24T02:47:09,331 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429331 2024-11-24T02:47:09,335 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,335 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,335 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,335 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,335 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,336 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429331 2024-11-24T02:47:09,336 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,336 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=179, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,336 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,336 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,336 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,336 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429331) roll requested 2024-11-24T02:47:09,337 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429336 2024-11-24T02:47:09,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429331 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429336 2024-11-24T02:47:09,342 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,342 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=180, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,342 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,342 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,342 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,342 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429336) roll requested 2024-11-24T02:47:09,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429342 2024-11-24T02:47:09,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,348 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429336 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429342 2024-11-24T02:47:09,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=181, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,348 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429342) roll requested 2024-11-24T02:47:09,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429349 2024-11-24T02:47:09,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,354 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,354 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,354 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,354 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,354 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429342 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429349 2024-11-24T02:47:09,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=182, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,355 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,355 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,355 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429349) roll requested 2024-11-24T02:47:09,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429355 2024-11-24T02:47:09,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429355 2024-11-24T02:47:09,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=183, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429355) roll requested 2024-11-24T02:47:09,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429361 2024-11-24T02:47:09,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429355 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429361 2024-11-24T02:47:09,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=184, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429361) roll requested 2024-11-24T02:47:09,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429367 2024-11-24T02:47:09,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429367 2024-11-24T02:47:09,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,372 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=185, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,372 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,372 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429367) roll requested 2024-11-24T02:47:09,372 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429372 2024-11-24T02:47:09,377 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,377 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,377 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,378 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,378 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,378 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429372 2024-11-24T02:47:09,378 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=186, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,378 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429372) roll requested 2024-11-24T02:47:09,379 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429378 2024-11-24T02:47:09,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,384 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429372 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429378 2024-11-24T02:47:09,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=187, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429378) roll requested 2024-11-24T02:47:09,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429384 2024-11-24T02:47:09,389 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,389 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429384 2024-11-24T02:47:09,390 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,390 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=188, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,390 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,390 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,390 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,390 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429384) roll requested 2024-11-24T02:47:09,391 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429390 2024-11-24T02:47:09,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429390 2024-11-24T02:47:09,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=189, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429390) roll requested 2024-11-24T02:47:09,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429397 2024-11-24T02:47:09,404 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,404 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,404 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,404 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,404 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429390 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429397 2024-11-24T02:47:09,405 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,405 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=190, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,405 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,405 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,405 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,405 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429397) roll requested 2024-11-24T02:47:09,405 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429405 2024-11-24T02:47:09,410 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,410 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,410 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,410 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,410 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,411 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429405 2024-11-24T02:47:09,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=191, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,411 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,411 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429405) roll requested 2024-11-24T02:47:09,411 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429411 2024-11-24T02:47:09,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429405 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429411 2024-11-24T02:47:09,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=192, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429411) roll requested 2024-11-24T02:47:09,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429416 2024-11-24T02:47:09,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429411 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429416 2024-11-24T02:47:09,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=193, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429416) roll requested 2024-11-24T02:47:09,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429423 2024-11-24T02:47:09,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429423 2024-11-24T02:47:09,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=194, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429423) roll requested 2024-11-24T02:47:09,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429428 2024-11-24T02:47:09,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429428 2024-11-24T02:47:09,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=195, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429428) roll requested 2024-11-24T02:47:09,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429436 2024-11-24T02:47:09,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429436 2024-11-24T02:47:09,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,442 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=196, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,442 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,442 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,442 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,442 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429436) roll requested 2024-11-24T02:47:09,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429442 2024-11-24T02:47:09,447 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,447 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,447 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,447 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,447 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,447 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429442 2024-11-24T02:47:09,448 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,448 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=197, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,448 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,448 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,448 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,448 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429442) roll requested 2024-11-24T02:47:09,448 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429448 2024-11-24T02:47:09,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429442 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429448 2024-11-24T02:47:09,455 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,456 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=198, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,456 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,456 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,456 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,456 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429448) roll requested 2024-11-24T02:47:09,456 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429456 2024-11-24T02:47:09,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429448 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429456 2024-11-24T02:47:09,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=199, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429456) roll requested 2024-11-24T02:47:09,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429461 2024-11-24T02:47:09,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,466 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429456 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429461 2024-11-24T02:47:09,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=200, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429461) roll requested 2024-11-24T02:47:09,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429466 2024-11-24T02:47:09,471 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429466 2024-11-24T02:47:09,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=201, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429466) roll requested 2024-11-24T02:47:09,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429473 2024-11-24T02:47:09,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429473 2024-11-24T02:47:09,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=202, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429473) roll requested 2024-11-24T02:47:09,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429479 2024-11-24T02:47:09,486 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,486 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,486 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,486 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,486 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,486 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429479 2024-11-24T02:47:09,487 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,487 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=203, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,487 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,487 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,487 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,487 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429479) roll requested 2024-11-24T02:47:09,488 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429487 2024-11-24T02:47:09,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429487 2024-11-24T02:47:09,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=204, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,496 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,496 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429487) roll requested 2024-11-24T02:47:09,496 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429496 2024-11-24T02:47:09,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,504 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,504 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429487 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429496 2024-11-24T02:47:09,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=205, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429496) roll requested 2024-11-24T02:47:09,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429505 2024-11-24T02:47:09,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,511 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,511 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,511 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429496 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429505 2024-11-24T02:47:09,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=206, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429505) roll requested 2024-11-24T02:47:09,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429512 2024-11-24T02:47:09,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429512 2024-11-24T02:47:09,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=207, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,521 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,521 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429512) roll requested 2024-11-24T02:47:09,521 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429521 2024-11-24T02:47:09,529 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,529 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429521 2024-11-24T02:47:09,530 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=208, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429521) roll requested 2024-11-24T02:47:09,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429531 2024-11-24T02:47:09,537 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,537 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,537 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,537 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429521 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429531 2024-11-24T02:47:09,538 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,538 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=209, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,538 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,538 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429531) roll requested 2024-11-24T02:47:09,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429538 2024-11-24T02:47:09,542 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,542 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,543 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429538 2024-11-24T02:47:09,543 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=210, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,543 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,543 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429538) roll requested 2024-11-24T02:47:09,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429544 2024-11-24T02:47:09,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,550 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429544 2024-11-24T02:47:09,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=211, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429544) roll requested 2024-11-24T02:47:09,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429551 2024-11-24T02:47:09,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429551 2024-11-24T02:47:09,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=212, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429551) roll requested 2024-11-24T02:47:09,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429559 2024-11-24T02:47:09,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429559 2024-11-24T02:47:09,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=213, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429559) roll requested 2024-11-24T02:47:09,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429568 2024-11-24T02:47:09,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429568 2024-11-24T02:47:09,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=214, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429568) roll requested 2024-11-24T02:47:09,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429576 2024-11-24T02:47:09,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429576 2024-11-24T02:47:09,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=215, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429576) roll requested 2024-11-24T02:47:09,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429584 2024-11-24T02:47:09,589 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,589 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,589 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,589 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,589 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,589 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429584 2024-11-24T02:47:09,590 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,590 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,590 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=216, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,590 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,590 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,590 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429584) roll requested 2024-11-24T02:47:09,590 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429590 2024-11-24T02:47:09,595 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,596 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,596 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,596 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,596 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,596 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429590 2024-11-24T02:47:09,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=217, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,597 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,597 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429590) roll requested 2024-11-24T02:47:09,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429597 2024-11-24T02:47:09,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429590 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429597 2024-11-24T02:47:09,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=218, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,604 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,604 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429597) roll requested 2024-11-24T02:47:09,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429604 2024-11-24T02:47:09,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,611 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,611 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429604 2024-11-24T02:47:09,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,612 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=219, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,612 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,612 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429604) roll requested 2024-11-24T02:47:09,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429613 2024-11-24T02:47:09,619 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,619 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429613 2024-11-24T02:47:09,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=220, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429613) roll requested 2024-11-24T02:47:09,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429620 2024-11-24T02:47:09,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,627 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,627 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429620 2024-11-24T02:47:09,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=221, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429620) roll requested 2024-11-24T02:47:09,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429628 2024-11-24T02:47:09,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,632 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429628 2024-11-24T02:47:09,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,633 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,633 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=222, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,633 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429628) roll requested 2024-11-24T02:47:09,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429634 2024-11-24T02:47:09,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429634 2024-11-24T02:47:09,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=223, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429634) roll requested 2024-11-24T02:47:09,640 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429640 2024-11-24T02:47:09,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429640 2024-11-24T02:47:09,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=224, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429640) roll requested 2024-11-24T02:47:09,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429649 2024-11-24T02:47:09,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429649 2024-11-24T02:47:09,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=225, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,659 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,659 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,659 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429649) roll requested 2024-11-24T02:47:09,659 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429659 2024-11-24T02:47:09,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429659 2024-11-24T02:47:09,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,667 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,667 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=226, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,667 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,667 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429659) roll requested 2024-11-24T02:47:09,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429667 2024-11-24T02:47:09,672 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,672 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,672 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,672 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,672 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,672 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429659 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429667 2024-11-24T02:47:09,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=227, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,673 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,673 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429667) roll requested 2024-11-24T02:47:09,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429673 2024-11-24T02:47:09,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429667 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429673 2024-11-24T02:47:09,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=228, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429673) roll requested 2024-11-24T02:47:09,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429678 2024-11-24T02:47:09,683 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,683 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,683 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,683 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429673 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429678 2024-11-24T02:47:09,684 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,684 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=229, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,684 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,684 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429678) roll requested 2024-11-24T02:47:09,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429685 2024-11-24T02:47:09,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429685 2024-11-24T02:47:09,695 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=230, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429685) roll requested 2024-11-24T02:47:09,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429696 2024-11-24T02:47:09,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,703 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,703 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429696 2024-11-24T02:47:09,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,704 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,704 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=231, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,704 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,704 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,704 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429696) roll requested 2024-11-24T02:47:09,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429704 2024-11-24T02:47:09,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429704 2024-11-24T02:47:09,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=232, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429704) roll requested 2024-11-24T02:47:09,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429710 2024-11-24T02:47:09,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429704 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429710 2024-11-24T02:47:09,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=233, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429710) roll requested 2024-11-24T02:47:09,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429717 2024-11-24T02:47:09,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429717 2024-11-24T02:47:09,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=234, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429717) roll requested 2024-11-24T02:47:09,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429725 2024-11-24T02:47:09,729 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,729 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429725 2024-11-24T02:47:09,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=235, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429725) roll requested 2024-11-24T02:47:09,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429732 2024-11-24T02:47:09,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429732 2024-11-24T02:47:09,738 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,738 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=236, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,738 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,738 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429732) roll requested 2024-11-24T02:47:09,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429738 2024-11-24T02:47:09,743 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,743 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,743 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429738 2024-11-24T02:47:09,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=237, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429738) roll requested 2024-11-24T02:47:09,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429744 2024-11-24T02:47:09,751 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,751 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,751 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,751 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,751 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,751 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429744 2024-11-24T02:47:09,752 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,752 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=238, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,752 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,752 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,752 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,752 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429744) roll requested 2024-11-24T02:47:09,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429752 2024-11-24T02:47:09,756 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429752 2024-11-24T02:47:09,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=239, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429752) roll requested 2024-11-24T02:47:09,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429758 2024-11-24T02:47:09,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429752 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429758 2024-11-24T02:47:09,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=240, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429758) roll requested 2024-11-24T02:47:09,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429764 2024-11-24T02:47:09,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429764 2024-11-24T02:47:09,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=241, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429764) roll requested 2024-11-24T02:47:09,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429770 2024-11-24T02:47:09,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429770 2024-11-24T02:47:09,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=242, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429770) roll requested 2024-11-24T02:47:09,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429776 2024-11-24T02:47:09,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429776 2024-11-24T02:47:09,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=243, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429776) roll requested 2024-11-24T02:47:09,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429782 2024-11-24T02:47:09,786 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,786 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429782 2024-11-24T02:47:09,787 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,787 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,787 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=244, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,788 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429782) roll requested 2024-11-24T02:47:09,788 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429788 2024-11-24T02:47:09,793 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,794 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,794 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,794 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,794 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429788 2024-11-24T02:47:09,794 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,794 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=245, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,795 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,795 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429788) roll requested 2024-11-24T02:47:09,795 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429795 2024-11-24T02:47:09,799 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,799 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,799 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,799 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429788 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429795 2024-11-24T02:47:09,800 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,800 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,800 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=246, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,800 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429795) roll requested 2024-11-24T02:47:09,801 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429801 2024-11-24T02:47:09,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,806 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,806 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,806 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429795 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429801 2024-11-24T02:47:09,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,807 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,807 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=247, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,807 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,807 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,807 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429801) roll requested 2024-11-24T02:47:09,807 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429807 2024-11-24T02:47:09,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429801 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429807 2024-11-24T02:47:09,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=248, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,815 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,815 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429807) roll requested 2024-11-24T02:47:09,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429815 2024-11-24T02:47:09,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429807 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429815 2024-11-24T02:47:09,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=249, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429815) roll requested 2024-11-24T02:47:09,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429823 2024-11-24T02:47:09,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,831 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429823 2024-11-24T02:47:09,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=250, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,832 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,832 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429823) roll requested 2024-11-24T02:47:09,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429832 2024-11-24T02:47:09,840 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,840 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,840 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,840 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,840 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,840 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429832 2024-11-24T02:47:09,841 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=251, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,841 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429832) roll requested 2024-11-24T02:47:09,842 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429842 2024-11-24T02:47:09,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,849 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,849 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,849 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429842 2024-11-24T02:47:09,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=252, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429842) roll requested 2024-11-24T02:47:09,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429851 2024-11-24T02:47:09,859 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,859 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429851 2024-11-24T02:47:09,860 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,860 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=253, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,861 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,861 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429851) roll requested 2024-11-24T02:47:09,861 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429861 2024-11-24T02:47:09,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429861 2024-11-24T02:47:09,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=254, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,871 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,871 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,871 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429861) roll requested 2024-11-24T02:47:09,871 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429871 2024-11-24T02:47:09,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429861 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429871 2024-11-24T02:47:09,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=255, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429871) roll requested 2024-11-24T02:47:09,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429881 2024-11-24T02:47:09,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,889 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,889 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429871 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429881 2024-11-24T02:47:09,890 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=256, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,890 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,890 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429881) roll requested 2024-11-24T02:47:09,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429890 2024-11-24T02:47:09,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429890 2024-11-24T02:47:09,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=257, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429890) roll requested 2024-11-24T02:47:09,900 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429900 2024-11-24T02:47:09,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429900 2024-11-24T02:47:09,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=258, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429900) roll requested 2024-11-24T02:47:09,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429907 2024-11-24T02:47:09,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,914 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,914 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429900 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429907 2024-11-24T02:47:09,915 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,915 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=259, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,915 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,915 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,916 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429907) roll requested 2024-11-24T02:47:09,916 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429916 2024-11-24T02:47:09,923 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,923 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,923 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,923 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,923 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429916 2024-11-24T02:47:09,924 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=260, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,924 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,924 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429916) roll requested 2024-11-24T02:47:09,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429924 2024-11-24T02:47:09,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429916 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429924 2024-11-24T02:47:09,930 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=261, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429924) roll requested 2024-11-24T02:47:09,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429931 2024-11-24T02:47:09,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429931 2024-11-24T02:47:09,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=262, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,938 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429931) roll requested 2024-11-24T02:47:09,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429938 2024-11-24T02:47:09,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429938 2024-11-24T02:47:09,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=263, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,947 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,947 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429938) roll requested 2024-11-24T02:47:09,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429947 2024-11-24T02:47:09,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429947 2024-11-24T02:47:09,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=264, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429947) roll requested 2024-11-24T02:47:09,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429955 2024-11-24T02:47:09,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429955 2024-11-24T02:47:09,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=265, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429955) roll requested 2024-11-24T02:47:09,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429964 2024-11-24T02:47:09,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429964 2024-11-24T02:47:09,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=266, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429964) roll requested 2024-11-24T02:47:09,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429972 2024-11-24T02:47:09,978 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,978 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,979 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,979 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429972 2024-11-24T02:47:09,979 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,980 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=267, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,980 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,980 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,980 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,980 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429972) roll requested 2024-11-24T02:47:09,980 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429980 2024-11-24T02:47:09,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429980 2024-11-24T02:47:09,989 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,989 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=268, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,989 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,990 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,990 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429980) roll requested 2024-11-24T02:47:09,990 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429990 2024-11-24T02:47:09,996 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,996 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,996 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,996 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,996 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:09,996 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429980 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429990 2024-11-24T02:47:09,997 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:09,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=269, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:09,997 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:09,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:09,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429990) roll requested 2024-11-24T02:47:09,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416429997 2024-11-24T02:47:10,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,003 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,003 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,003 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,003 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,003 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429990 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429997 2024-11-24T02:47:10,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,004 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=270, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,004 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,004 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,004 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,004 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416429997) roll requested 2024-11-24T02:47:10,004 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430004 2024-11-24T02:47:10,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,012 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,012 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416429997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430004 2024-11-24T02:47:10,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=271, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430004) roll requested 2024-11-24T02:47:10,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430013 2024-11-24T02:47:10,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,020 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,020 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,020 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430004 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430013 2024-11-24T02:47:10,021 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=272, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,021 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,021 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430013) roll requested 2024-11-24T02:47:10,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430021 2024-11-24T02:47:10,028 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,028 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,028 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,029 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,029 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,029 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430021 2024-11-24T02:47:10,029 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,030 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,030 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=273, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,030 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,030 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,030 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430021) roll requested 2024-11-24T02:47:10,030 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430030 2024-11-24T02:47:10,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430030 2024-11-24T02:47:10,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=274, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430030) roll requested 2024-11-24T02:47:10,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430040 2024-11-24T02:47:10,047 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,047 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,047 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,048 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,048 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,048 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430030 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430040 2024-11-24T02:47:10,049 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,049 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,049 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=275, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,049 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,049 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,049 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430040) roll requested 2024-11-24T02:47:10,049 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430049 2024-11-24T02:47:10,057 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,057 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,057 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,057 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430049 2024-11-24T02:47:10,059 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,059 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,059 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=276, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,059 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,059 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,059 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430049) roll requested 2024-11-24T02:47:10,059 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430059 2024-11-24T02:47:10,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430049 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430059 2024-11-24T02:47:10,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,069 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=277, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,069 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,069 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,069 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430059) roll requested 2024-11-24T02:47:10,069 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430069 2024-11-24T02:47:10,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,077 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,077 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430059 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430069 2024-11-24T02:47:10,079 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=278, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430069) roll requested 2024-11-24T02:47:10,080 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430079 2024-11-24T02:47:10,087 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,087 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,087 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,087 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,088 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430069 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430079 2024-11-24T02:47:10,088 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,088 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,088 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=279, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,088 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,089 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,089 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430079) roll requested 2024-11-24T02:47:10,089 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430089 2024-11-24T02:47:10,096 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,096 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,096 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,096 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,096 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,096 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430089 2024-11-24T02:47:10,097 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,097 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=280, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,097 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430089) roll requested 2024-11-24T02:47:10,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430098 2024-11-24T02:47:10,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,106 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430089 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430098 2024-11-24T02:47:10,107 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,107 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,107 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=281, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,107 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,107 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,107 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430098) roll requested 2024-11-24T02:47:10,107 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430107 2024-11-24T02:47:10,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430107 2024-11-24T02:47:10,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,115 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=282, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,115 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,115 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,115 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430107) roll requested 2024-11-24T02:47:10,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430115 2024-11-24T02:47:10,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430107 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430115 2024-11-24T02:47:10,121 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,121 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=283, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430115) roll requested 2024-11-24T02:47:10,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430121 2024-11-24T02:47:10,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430115 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430121 2024-11-24T02:47:10,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=284, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430121) roll requested 2024-11-24T02:47:10,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430129 2024-11-24T02:47:10,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430129 2024-11-24T02:47:10,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=285, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430129) roll requested 2024-11-24T02:47:10,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430136 2024-11-24T02:47:10,141 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,141 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430136 2024-11-24T02:47:10,142 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,142 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=286, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430136) roll requested 2024-11-24T02:47:10,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430143 2024-11-24T02:47:10,148 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,148 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,148 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430143 2024-11-24T02:47:10,149 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,149 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,149 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=287, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,149 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430143) roll requested 2024-11-24T02:47:10,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430149 2024-11-24T02:47:10,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430149 2024-11-24T02:47:10,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=288, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430149) roll requested 2024-11-24T02:47:10,157 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430157 2024-11-24T02:47:10,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,163 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430157 2024-11-24T02:47:10,164 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,164 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,164 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=289, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,164 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430157) roll requested 2024-11-24T02:47:10,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430165 2024-11-24T02:47:10,172 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,172 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,172 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,172 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,172 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430165 2024-11-24T02:47:10,173 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,173 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=290, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430165) roll requested 2024-11-24T02:47:10,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430174 2024-11-24T02:47:10,182 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,182 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,182 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,182 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,182 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,183 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430174 2024-11-24T02:47:10,183 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,184 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,184 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=291, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,184 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,184 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,184 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430174) roll requested 2024-11-24T02:47:10,184 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430184 2024-11-24T02:47:10,189 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,189 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,189 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,189 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,190 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,190 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430184 2024-11-24T02:47:10,190 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,190 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,190 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=292, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,190 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,191 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,191 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430184) roll requested 2024-11-24T02:47:10,191 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430191 2024-11-24T02:47:10,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430184 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430191 2024-11-24T02:47:10,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,198 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=293, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,198 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430191) roll requested 2024-11-24T02:47:10,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430198 2024-11-24T02:47:10,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,205 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,205 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,205 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,205 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,205 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430191 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430198 2024-11-24T02:47:10,206 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,206 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,206 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,206 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=294, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430198) roll requested 2024-11-24T02:47:10,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430207 2024-11-24T02:47:10,213 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,213 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,213 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,213 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,213 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,213 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430207 2024-11-24T02:47:10,214 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,214 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,214 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,214 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=295, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,214 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,214 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430207) roll requested 2024-11-24T02:47:10,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430214 2024-11-24T02:47:10,220 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,220 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,220 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,220 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,220 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430214 2024-11-24T02:47:10,221 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=296, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,222 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,222 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430214) roll requested 2024-11-24T02:47:10,222 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430222 2024-11-24T02:47:10,229 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,229 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,230 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430214 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430222 2024-11-24T02:47:10,230 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,230 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,231 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=297, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,231 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,231 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430222) roll requested 2024-11-24T02:47:10,231 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430231 2024-11-24T02:47:10,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430222 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430231 2024-11-24T02:47:10,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=298, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,241 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,241 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430231) roll requested 2024-11-24T02:47:10,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430241 2024-11-24T02:47:10,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430231 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430241 2024-11-24T02:47:10,250 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,250 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=299, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430241) roll requested 2024-11-24T02:47:10,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430250 2024-11-24T02:47:10,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,256 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430241 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430250 2024-11-24T02:47:10,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=300, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430250) roll requested 2024-11-24T02:47:10,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430257 2024-11-24T02:47:10,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430257 2024-11-24T02:47:10,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,264 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=301, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,264 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,264 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430257) roll requested 2024-11-24T02:47:10,264 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430264 2024-11-24T02:47:10,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430264 2024-11-24T02:47:10,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=302, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430264) roll requested 2024-11-24T02:47:10,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430272 2024-11-24T02:47:10,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430264 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430272 2024-11-24T02:47:10,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=303, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430272) roll requested 2024-11-24T02:47:10,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430281 2024-11-24T02:47:10,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,288 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430281 2024-11-24T02:47:10,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,289 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=304, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,289 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430281) roll requested 2024-11-24T02:47:10,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430289 2024-11-24T02:47:10,295 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430289 2024-11-24T02:47:10,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=305, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430289) roll requested 2024-11-24T02:47:10,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430297 2024-11-24T02:47:10,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430297 2024-11-24T02:47:10,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,306 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,306 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=306, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,306 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,306 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,306 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430297) roll requested 2024-11-24T02:47:10,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430306 2024-11-24T02:47:10,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430306 2024-11-24T02:47:10,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=307, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430306) roll requested 2024-11-24T02:47:10,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430314 2024-11-24T02:47:10,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430306 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430314 2024-11-24T02:47:10,321 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,321 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=308, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,321 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430314) roll requested 2024-11-24T02:47:10,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430321 2024-11-24T02:47:10,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430321 2024-11-24T02:47:10,329 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,329 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,329 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=309, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,329 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,329 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,329 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430321) roll requested 2024-11-24T02:47:10,329 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430329 2024-11-24T02:47:10,336 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,337 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,337 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,337 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,337 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430329 2024-11-24T02:47:10,338 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,338 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,338 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=310, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,338 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430329) roll requested 2024-11-24T02:47:10,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430339 2024-11-24T02:47:10,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,347 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,347 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,347 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,347 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,347 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430329 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430339 2024-11-24T02:47:10,348 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,348 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,348 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,348 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=311, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,349 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,349 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430339) roll requested 2024-11-24T02:47:10,349 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430349 2024-11-24T02:47:10,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430349 2024-11-24T02:47:10,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=312, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430349) roll requested 2024-11-24T02:47:10,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430356 2024-11-24T02:47:10,362 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,362 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,363 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430349 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430356 2024-11-24T02:47:10,364 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,364 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,364 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=313, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,364 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,364 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,365 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430356) roll requested 2024-11-24T02:47:10,365 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430365 2024-11-24T02:47:10,372 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,372 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430356 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430365 2024-11-24T02:47:10,373 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=314, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430365) roll requested 2024-11-24T02:47:10,374 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430374 2024-11-24T02:47:10,379 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,379 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430365 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430374 2024-11-24T02:47:10,380 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=315, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430374) roll requested 2024-11-24T02:47:10,381 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430381 2024-11-24T02:47:10,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430374 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430381 2024-11-24T02:47:10,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=316, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430381) roll requested 2024-11-24T02:47:10,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430389 2024-11-24T02:47:10,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,398 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,398 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,398 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,398 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430381 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430389 2024-11-24T02:47:10,399 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=317, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430389) roll requested 2024-11-24T02:47:10,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430400 2024-11-24T02:47:10,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,406 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,406 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,406 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,406 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,406 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430400 2024-11-24T02:47:10,407 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,407 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,407 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=318, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430400) roll requested 2024-11-24T02:47:10,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430407 2024-11-24T02:47:10,411 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,411 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,411 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,411 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,411 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,412 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430407 2024-11-24T02:47:10,412 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,412 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,412 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,412 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=319, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430407) roll requested 2024-11-24T02:47:10,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430412 2024-11-24T02:47:10,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,417 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,417 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,417 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430412 2024-11-24T02:47:10,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,418 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=320, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,418 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,418 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,418 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430412) roll requested 2024-11-24T02:47:10,418 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430418 2024-11-24T02:47:10,423 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,424 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,424 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,424 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,424 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,424 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430418 2024-11-24T02:47:10,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=321, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430418) roll requested 2024-11-24T02:47:10,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430425 2024-11-24T02:47:10,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,431 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,431 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,431 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430418 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430425 2024-11-24T02:47:10,432 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,432 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=322, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430425) roll requested 2024-11-24T02:47:10,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430433 2024-11-24T02:47:10,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,441 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,441 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,441 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,441 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430433 2024-11-24T02:47:10,442 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=323, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,443 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,443 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430433) roll requested 2024-11-24T02:47:10,443 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430443 2024-11-24T02:47:10,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,450 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430443 2024-11-24T02:47:10,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,451 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=324, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,451 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430443) roll requested 2024-11-24T02:47:10,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430452 2024-11-24T02:47:10,458 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,458 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,458 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430443 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430452 2024-11-24T02:47:10,459 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,459 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,459 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=325, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,460 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,460 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430452) roll requested 2024-11-24T02:47:10,460 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430460 2024-11-24T02:47:10,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,467 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,467 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,467 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,467 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430460 2024-11-24T02:47:10,468 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,468 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,468 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=326, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,468 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,468 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430460) roll requested 2024-11-24T02:47:10,469 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430469 2024-11-24T02:47:10,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430460 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430469 2024-11-24T02:47:10,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=327, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430469) roll requested 2024-11-24T02:47:10,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430477 2024-11-24T02:47:10,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430469 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430477 2024-11-24T02:47:10,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=328, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430477) roll requested 2024-11-24T02:47:10,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430485 2024-11-24T02:47:10,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,492 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,492 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430485 2024-11-24T02:47:10,492 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,492 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=329, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,493 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,493 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430485) roll requested 2024-11-24T02:47:10,493 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430493 2024-11-24T02:47:10,499 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,500 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,500 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430493 2024-11-24T02:47:10,500 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,500 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,501 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,501 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=330, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,501 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,501 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430493) roll requested 2024-11-24T02:47:10,501 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430501 2024-11-24T02:47:10,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430493 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430501 2024-11-24T02:47:10,508 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=331, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,508 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,508 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430501) roll requested 2024-11-24T02:47:10,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430509 2024-11-24T02:47:10,515 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430501 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430509 2024-11-24T02:47:10,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=332, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430509) roll requested 2024-11-24T02:47:10,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430516 2024-11-24T02:47:10,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,524 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430516 2024-11-24T02:47:10,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,525 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=333, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,525 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,525 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,525 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430516) roll requested 2024-11-24T02:47:10,525 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430525 2024-11-24T02:47:10,531 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,531 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,531 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,531 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,531 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430525 2024-11-24T02:47:10,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,532 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=334, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430525) roll requested 2024-11-24T02:47:10,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430533 2024-11-24T02:47:10,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430525 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430533 2024-11-24T02:47:10,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=335, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430533) roll requested 2024-11-24T02:47:10,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430541 2024-11-24T02:47:10,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430541 2024-11-24T02:47:10,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=336, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430541) roll requested 2024-11-24T02:47:10,551 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430550 2024-11-24T02:47:10,558 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,558 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,558 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,558 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,558 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,558 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430550 2024-11-24T02:47:10,559 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,559 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=337, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,559 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430550) roll requested 2024-11-24T02:47:10,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430559 2024-11-24T02:47:10,565 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,565 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,566 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,566 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,566 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430559 2024-11-24T02:47:10,566 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,567 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,567 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=338, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,567 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,567 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,567 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430559) roll requested 2024-11-24T02:47:10,567 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430567 2024-11-24T02:47:10,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430567 2024-11-24T02:47:10,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=339, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430567) roll requested 2024-11-24T02:47:10,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430575 2024-11-24T02:47:10,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,582 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,582 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,582 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430567 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430575 2024-11-24T02:47:10,583 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,583 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=340, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430575) roll requested 2024-11-24T02:47:10,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430584 2024-11-24T02:47:10,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430584 2024-11-24T02:47:10,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=341, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430584) roll requested 2024-11-24T02:47:10,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430592 2024-11-24T02:47:10,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430592 2024-11-24T02:47:10,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=342, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,601 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430592) roll requested 2024-11-24T02:47:10,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430601 2024-11-24T02:47:10,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,610 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,610 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,610 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430601 2024-11-24T02:47:10,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,611 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,611 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=343, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,611 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,611 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,611 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430601) roll requested 2024-11-24T02:47:10,611 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430611 2024-11-24T02:47:10,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,618 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,618 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430611 2024-11-24T02:47:10,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=344, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430611) roll requested 2024-11-24T02:47:10,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430620 2024-11-24T02:47:10,624 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,624 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,624 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,624 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,624 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,624 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430611 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430620 2024-11-24T02:47:10,625 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,625 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,625 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=345, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,625 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,625 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,625 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430620) roll requested 2024-11-24T02:47:10,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430625 2024-11-24T02:47:10,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430625 2024-11-24T02:47:10,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=346, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430625) roll requested 2024-11-24T02:47:10,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430630 2024-11-24T02:47:10,635 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,635 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,635 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430625 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430630 2024-11-24T02:47:10,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=347, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430630) roll requested 2024-11-24T02:47:10,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430637 2024-11-24T02:47:10,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,643 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430637 2024-11-24T02:47:10,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,644 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,644 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=348, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,644 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430637) roll requested 2024-11-24T02:47:10,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430644 2024-11-24T02:47:10,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430644 2024-11-24T02:47:10,653 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=349, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,653 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,653 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430644) roll requested 2024-11-24T02:47:10,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430653 2024-11-24T02:47:10,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430653 2024-11-24T02:47:10,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=350, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,665 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,665 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430653) roll requested 2024-11-24T02:47:10,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430665 2024-11-24T02:47:10,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430665 2024-11-24T02:47:10,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=351, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430665) roll requested 2024-11-24T02:47:10,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430670 2024-11-24T02:47:10,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430670 2024-11-24T02:47:10,674 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=352, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430670) roll requested 2024-11-24T02:47:10,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430675 2024-11-24T02:47:10,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430675 2024-11-24T02:47:10,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=353, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,680 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430675) roll requested 2024-11-24T02:47:10,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430681 2024-11-24T02:47:10,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430681 2024-11-24T02:47:10,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=354, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430681) roll requested 2024-11-24T02:47:10,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430686 2024-11-24T02:47:10,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430686 2024-11-24T02:47:10,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=355, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430686) roll requested 2024-11-24T02:47:10,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430693 2024-11-24T02:47:10,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430693 2024-11-24T02:47:10,701 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,701 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=356, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430693) roll requested 2024-11-24T02:47:10,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430702 2024-11-24T02:47:10,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,709 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,709 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,709 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,709 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,709 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430702 2024-11-24T02:47:10,710 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,710 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,710 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=357, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,710 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,710 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,710 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430702) roll requested 2024-11-24T02:47:10,710 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430710 2024-11-24T02:47:10,716 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,716 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,716 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,716 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,716 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,716 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430710 2024-11-24T02:47:10,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,717 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=358, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430710) roll requested 2024-11-24T02:47:10,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430717 2024-11-24T02:47:10,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430710 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430717 2024-11-24T02:47:10,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=359, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,726 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430717) roll requested 2024-11-24T02:47:10,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430726 2024-11-24T02:47:10,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430726 2024-11-24T02:47:10,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=360, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430726) roll requested 2024-11-24T02:47:10,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430734 2024-11-24T02:47:10,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430734 2024-11-24T02:47:10,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=361, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430734) roll requested 2024-11-24T02:47:10,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430744 2024-11-24T02:47:10,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430744 2024-11-24T02:47:10,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=362, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430744) roll requested 2024-11-24T02:47:10,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430751 2024-11-24T02:47:10,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,757 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,757 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430751 2024-11-24T02:47:10,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=363, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430751) roll requested 2024-11-24T02:47:10,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430758 2024-11-24T02:47:10,764 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,764 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,765 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430758 2024-11-24T02:47:10,765 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=364, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,766 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,766 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430758) roll requested 2024-11-24T02:47:10,766 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430766 2024-11-24T02:47:10,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,773 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,773 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,773 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,773 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,773 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430766 2024-11-24T02:47:10,774 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,774 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,774 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=365, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,774 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,774 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,774 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430766) roll requested 2024-11-24T02:47:10,774 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430774 2024-11-24T02:47:10,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430766 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430774 2024-11-24T02:47:10,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=366, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430774) roll requested 2024-11-24T02:47:10,783 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430783 2024-11-24T02:47:10,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430774 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430783 2024-11-24T02:47:10,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=367, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430783) roll requested 2024-11-24T02:47:10,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430792 2024-11-24T02:47:10,800 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,800 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,800 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,800 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,800 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,801 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430783 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430792 2024-11-24T02:47:10,802 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,802 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,802 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=368, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,802 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,802 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,802 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430792) roll requested 2024-11-24T02:47:10,802 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430802 2024-11-24T02:47:10,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,811 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,811 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,811 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,811 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430802 2024-11-24T02:47:10,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=369, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430802) roll requested 2024-11-24T02:47:10,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430813 2024-11-24T02:47:10,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,819 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,820 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430802 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430813 2024-11-24T02:47:10,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,821 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,821 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,821 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=370, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,821 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,821 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430813) roll requested 2024-11-24T02:47:10,821 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430821 2024-11-24T02:47:10,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430821 2024-11-24T02:47:10,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=371, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430821) roll requested 2024-11-24T02:47:10,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430829 2024-11-24T02:47:10,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,837 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430821 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430829 2024-11-24T02:47:10,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,839 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=372, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430829) roll requested 2024-11-24T02:47:10,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430839 2024-11-24T02:47:10,847 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,847 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430839 2024-11-24T02:47:10,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,849 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=373, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,849 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,849 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430839) roll requested 2024-11-24T02:47:10,849 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430849 2024-11-24T02:47:10,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430849 2024-11-24T02:47:10,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=374, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430849) roll requested 2024-11-24T02:47:10,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430858 2024-11-24T02:47:10,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430849 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430858 2024-11-24T02:47:10,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=375, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430858) roll requested 2024-11-24T02:47:10,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430866 2024-11-24T02:47:10,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,874 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,874 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430866 2024-11-24T02:47:10,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,875 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=376, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,875 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,875 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430866) roll requested 2024-11-24T02:47:10,875 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430875 2024-11-24T02:47:10,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430875 2024-11-24T02:47:10,882 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,882 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,882 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=377, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,882 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,883 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430875) roll requested 2024-11-24T02:47:10,883 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430883 2024-11-24T02:47:10,889 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,890 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430875 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430883 2024-11-24T02:47:10,891 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,891 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,891 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,891 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=378, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,891 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,891 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430883) roll requested 2024-11-24T02:47:10,891 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430891 2024-11-24T02:47:10,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430883 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430891 2024-11-24T02:47:10,897 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,897 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=379, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,897 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,898 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,898 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430891) roll requested 2024-11-24T02:47:10,898 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430898 2024-11-24T02:47:10,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,904 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,904 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,904 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430891 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430898 2024-11-24T02:47:10,905 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,905 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,905 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=380, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,905 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,905 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,905 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430898) roll requested 2024-11-24T02:47:10,905 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430905 2024-11-24T02:47:10,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430898 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430905 2024-11-24T02:47:10,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=381, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430905) roll requested 2024-11-24T02:47:10,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430913 2024-11-24T02:47:10,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430905 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430913 2024-11-24T02:47:10,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=382, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430913) roll requested 2024-11-24T02:47:10,921 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430920 2024-11-24T02:47:10,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430920 2024-11-24T02:47:10,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=383, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430920) roll requested 2024-11-24T02:47:10,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430928 2024-11-24T02:47:10,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430928 2024-11-24T02:47:10,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=384, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430928) roll requested 2024-11-24T02:47:10,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430937 2024-11-24T02:47:10,945 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,945 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430937 2024-11-24T02:47:10,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=385, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430937) roll requested 2024-11-24T02:47:10,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430946 2024-11-24T02:47:10,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430946 2024-11-24T02:47:10,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=386, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430946) roll requested 2024-11-24T02:47:10,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430952 2024-11-24T02:47:10,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430952 2024-11-24T02:47:10,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=387, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430952) roll requested 2024-11-24T02:47:10,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430961 2024-11-24T02:47:10,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430961 2024-11-24T02:47:10,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=388, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430961) roll requested 2024-11-24T02:47:10,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430967 2024-11-24T02:47:10,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430967 2024-11-24T02:47:10,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=389, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430967) roll requested 2024-11-24T02:47:10,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430972 2024-11-24T02:47:10,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,976 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,976 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430972 2024-11-24T02:47:10,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=390, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430972) roll requested 2024-11-24T02:47:10,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430977 2024-11-24T02:47:10,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430977 2024-11-24T02:47:10,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=391, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,984 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,984 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430977) roll requested 2024-11-24T02:47:10,984 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430984 2024-11-24T02:47:10,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430984 2024-11-24T02:47:10,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,991 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=392, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430984) roll requested 2024-11-24T02:47:10,992 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430992 2024-11-24T02:47:10,998 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:10,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430984 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430992 2024-11-24T02:47:10,998 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:10,998 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=393, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:10,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:10,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:10,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430992) roll requested 2024-11-24T02:47:10,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416430999 2024-11-24T02:47:11,003 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430999 2024-11-24T02:47:11,004 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=394, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416430999) roll requested 2024-11-24T02:47:11,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431005 2024-11-24T02:47:11,011 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,011 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,011 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416430999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431005 2024-11-24T02:47:11,012 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,012 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=395, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,012 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431005) roll requested 2024-11-24T02:47:11,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431012 2024-11-24T02:47:11,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,019 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,019 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431012 2024-11-24T02:47:11,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,020 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=396, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,020 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,020 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431012) roll requested 2024-11-24T02:47:11,020 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431020 2024-11-24T02:47:11,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431020 2024-11-24T02:47:11,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=397, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431020) roll requested 2024-11-24T02:47:11,027 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431027 2024-11-24T02:47:11,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,033 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,033 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,033 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431020 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431027 2024-11-24T02:47:11,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=398, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,034 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431027) roll requested 2024-11-24T02:47:11,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431035 2024-11-24T02:47:11,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431027 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431035 2024-11-24T02:47:11,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,045 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,045 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=399, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431035) roll requested 2024-11-24T02:47:11,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431045 2024-11-24T02:47:11,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,054 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,054 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431045 2024-11-24T02:47:11,055 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,055 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,055 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,055 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=400, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,055 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,055 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431045) roll requested 2024-11-24T02:47:11,056 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431055 2024-11-24T02:47:11,064 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431055 2024-11-24T02:47:11,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,066 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=401, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,066 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,066 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,066 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431055) roll requested 2024-11-24T02:47:11,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431066 2024-11-24T02:47:11,071 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,072 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,072 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,072 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,072 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,072 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431066 2024-11-24T02:47:11,073 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,073 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,073 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=402, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,073 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431066) roll requested 2024-11-24T02:47:11,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431073 2024-11-24T02:47:11,078 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431073 2024-11-24T02:47:11,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,079 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=403, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,079 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431073) roll requested 2024-11-24T02:47:11,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431079 2024-11-24T02:47:11,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431079 2024-11-24T02:47:11,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,086 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,086 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=404, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,086 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,086 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431079) roll requested 2024-11-24T02:47:11,086 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431086 2024-11-24T02:47:11,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431086 2024-11-24T02:47:11,094 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,094 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,094 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=405, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,094 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,095 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,095 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431086) roll requested 2024-11-24T02:47:11,095 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431095 2024-11-24T02:47:11,101 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,101 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,101 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,101 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431086 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431095 2024-11-24T02:47:11,102 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,102 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,102 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,103 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=406, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,103 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,103 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431095) roll requested 2024-11-24T02:47:11,103 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431103 2024-11-24T02:47:11,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431095 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431103 2024-11-24T02:47:11,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=407, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431103) roll requested 2024-11-24T02:47:11,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431110 2024-11-24T02:47:11,117 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431103 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431110 2024-11-24T02:47:11,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,119 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=408, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,119 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,119 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,119 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431110) roll requested 2024-11-24T02:47:11,119 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431119 2024-11-24T02:47:11,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,128 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431119 2024-11-24T02:47:11,129 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,129 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,129 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,129 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=409, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,129 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431119) roll requested 2024-11-24T02:47:11,130 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431130 2024-11-24T02:47:11,137 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,137 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,138 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,138 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,138 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431119 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431130 2024-11-24T02:47:11,139 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,139 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,139 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=410, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,139 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,139 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,139 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431130) roll requested 2024-11-24T02:47:11,139 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431139 2024-11-24T02:47:11,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431130 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431139 2024-11-24T02:47:11,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=411, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,148 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,148 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431139) roll requested 2024-11-24T02:47:11,148 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431148 2024-11-24T02:47:11,155 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,155 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,155 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431139 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431148 2024-11-24T02:47:11,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=412, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431148) roll requested 2024-11-24T02:47:11,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431157 2024-11-24T02:47:11,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431148 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431157 2024-11-24T02:47:11,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=413, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431157) roll requested 2024-11-24T02:47:11,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431165 2024-11-24T02:47:11,173 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,173 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,173 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,173 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,173 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,173 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431165 2024-11-24T02:47:11,174 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,174 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,174 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=414, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,174 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,174 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,174 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431165) roll requested 2024-11-24T02:47:11,174 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431174 2024-11-24T02:47:11,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,180 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,180 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,180 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,180 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431174 2024-11-24T02:47:11,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=415, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431174) roll requested 2024-11-24T02:47:11,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431181 2024-11-24T02:47:11,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431174 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431181 2024-11-24T02:47:11,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=416, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431181) roll requested 2024-11-24T02:47:11,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431188 2024-11-24T02:47:11,195 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,195 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,195 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,195 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,195 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,196 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431188 2024-11-24T02:47:11,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,197 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=417, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,197 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,197 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431188) roll requested 2024-11-24T02:47:11,197 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431197 2024-11-24T02:47:11,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431197 2024-11-24T02:47:11,205 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,205 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=418, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431197) roll requested 2024-11-24T02:47:11,206 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431205 2024-11-24T02:47:11,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,211 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431197 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431205 2024-11-24T02:47:11,212 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,212 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=419, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,212 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431205) roll requested 2024-11-24T02:47:11,213 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431213 2024-11-24T02:47:11,219 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,219 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,219 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,219 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,219 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,220 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431213 2024-11-24T02:47:11,220 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,221 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,221 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=420, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,221 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,221 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,221 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431213) roll requested 2024-11-24T02:47:11,221 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431221 2024-11-24T02:47:11,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431221 2024-11-24T02:47:11,229 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=421, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431221) roll requested 2024-11-24T02:47:11,230 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431229 2024-11-24T02:47:11,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,236 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,236 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,236 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,236 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431221 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431229 2024-11-24T02:47:11,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=422, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,237 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431229) roll requested 2024-11-24T02:47:11,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431237 2024-11-24T02:47:11,242 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,242 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,243 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,243 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,243 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,243 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431237 2024-11-24T02:47:11,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=423, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,244 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,244 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431237) roll requested 2024-11-24T02:47:11,244 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431244 2024-11-24T02:47:11,251 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,251 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,251 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,252 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,252 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,252 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431244 2024-11-24T02:47:11,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,253 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,253 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,253 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=424, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,253 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431244) roll requested 2024-11-24T02:47:11,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431253 2024-11-24T02:47:11,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,261 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431244 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431253 2024-11-24T02:47:11,262 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,262 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,262 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=425, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,262 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,262 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,262 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431253) roll requested 2024-11-24T02:47:11,262 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431262 2024-11-24T02:47:11,269 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,269 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,270 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,270 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,270 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,270 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431262 2024-11-24T02:47:11,271 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,271 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,271 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=426, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,271 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,271 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,271 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431262) roll requested 2024-11-24T02:47:11,271 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431271 2024-11-24T02:47:11,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431262 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431271 2024-11-24T02:47:11,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=427, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431271) roll requested 2024-11-24T02:47:11,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431281 2024-11-24T02:47:11,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,290 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,290 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431271 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431281 2024-11-24T02:47:11,291 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,291 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=428, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,291 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431281) roll requested 2024-11-24T02:47:11,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431291 2024-11-24T02:47:11,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431291 2024-11-24T02:47:11,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=429, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431291) roll requested 2024-11-24T02:47:11,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431301 2024-11-24T02:47:11,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,307 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431301 2024-11-24T02:47:11,308 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=430, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431301) roll requested 2024-11-24T02:47:11,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431309 2024-11-24T02:47:11,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431309 2024-11-24T02:47:11,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=431, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431309) roll requested 2024-11-24T02:47:11,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431316 2024-11-24T02:47:11,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,322 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431316 2024-11-24T02:47:11,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=432, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431316) roll requested 2024-11-24T02:47:11,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431323 2024-11-24T02:47:11,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,329 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,329 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,329 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,330 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431323 2024-11-24T02:47:11,331 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=433, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,332 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,332 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431323) roll requested 2024-11-24T02:47:11,332 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431332 2024-11-24T02:47:11,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,342 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,342 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,342 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,342 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431332 2024-11-24T02:47:11,343 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,343 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,343 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,343 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=434, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,343 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,343 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431332) roll requested 2024-11-24T02:47:11,343 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431343 2024-11-24T02:47:11,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431332 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431343 2024-11-24T02:47:11,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,351 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=435, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,351 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,351 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431343) roll requested 2024-11-24T02:47:11,351 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431351 2024-11-24T02:47:11,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,358 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,358 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,358 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,358 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431343 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431351 2024-11-24T02:47:11,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=436, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,359 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431351) roll requested 2024-11-24T02:47:11,359 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431359 2024-11-24T02:47:11,364 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,364 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,364 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,365 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,365 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,365 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431351 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431359 2024-11-24T02:47:11,365 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,365 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,365 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,366 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=437, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,366 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,366 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431359) roll requested 2024-11-24T02:47:11,366 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431366 2024-11-24T02:47:11,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,371 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,371 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,371 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,371 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431359 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431366 2024-11-24T02:47:11,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,373 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=438, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,373 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431366) roll requested 2024-11-24T02:47:11,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431373 2024-11-24T02:47:11,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,380 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431366 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431373 2024-11-24T02:47:11,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,381 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=439, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,381 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,381 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,381 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431373) roll requested 2024-11-24T02:47:11,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431382 2024-11-24T02:47:11,388 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,388 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,388 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,388 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,388 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,388 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431382 2024-11-24T02:47:11,389 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,389 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,389 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,389 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=440, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,389 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,389 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431382) roll requested 2024-11-24T02:47:11,389 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431389 2024-11-24T02:47:11,396 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,396 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,396 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,397 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,397 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,397 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431389 2024-11-24T02:47:11,397 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,398 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,398 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,398 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=441, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,398 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,398 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431389) roll requested 2024-11-24T02:47:11,398 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431398 2024-11-24T02:47:11,405 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,405 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,405 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,405 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,405 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,405 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431389 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431398 2024-11-24T02:47:11,406 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,406 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=442, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,406 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,407 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,407 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,407 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431398) roll requested 2024-11-24T02:47:11,407 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431407 2024-11-24T02:47:11,413 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,413 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,413 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,413 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,413 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,413 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431398 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431407 2024-11-24T02:47:11,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=443, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431407) roll requested 2024-11-24T02:47:11,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431415 2024-11-24T02:47:11,419 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,419 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,419 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,419 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,419 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,419 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431407 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431415 2024-11-24T02:47:11,420 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,420 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,420 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=444, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,420 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,420 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,420 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431415) roll requested 2024-11-24T02:47:11,420 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431420 2024-11-24T02:47:11,424 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,425 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,425 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,425 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,425 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,425 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431420 2024-11-24T02:47:11,425 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,425 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=445, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,425 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,426 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431420) roll requested 2024-11-24T02:47:11,426 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431426 2024-11-24T02:47:11,430 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,430 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431420 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431426 2024-11-24T02:47:11,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=446, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431426) roll requested 2024-11-24T02:47:11,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431431 2024-11-24T02:47:11,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,436 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,437 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431426 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431431 2024-11-24T02:47:11,437 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,437 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,437 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=447, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,437 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,438 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,438 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431431) roll requested 2024-11-24T02:47:11,438 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431438 2024-11-24T02:47:11,443 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,443 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,443 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,443 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431438 2024-11-24T02:47:11,444 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=448, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,444 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,444 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431438) roll requested 2024-11-24T02:47:11,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431445 2024-11-24T02:47:11,450 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,450 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,450 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431438 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431445 2024-11-24T02:47:11,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=449, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431445) roll requested 2024-11-24T02:47:11,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431452 2024-11-24T02:47:11,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,460 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,460 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431452 2024-11-24T02:47:11,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=450, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431452) roll requested 2024-11-24T02:47:11,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431461 2024-11-24T02:47:11,467 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,467 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431461 2024-11-24T02:47:11,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,469 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,469 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=451, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,469 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,469 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431461) roll requested 2024-11-24T02:47:11,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431470 2024-11-24T02:47:11,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431470 2024-11-24T02:47:11,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=452, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431470) roll requested 2024-11-24T02:47:11,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431477 2024-11-24T02:47:11,483 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,483 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,483 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431477 2024-11-24T02:47:11,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=453, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431477) roll requested 2024-11-24T02:47:11,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431485 2024-11-24T02:47:11,491 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,491 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,491 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,491 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,491 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431485 2024-11-24T02:47:11,493 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,493 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,493 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,493 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=454, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,494 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,494 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431485) roll requested 2024-11-24T02:47:11,494 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431494 2024-11-24T02:47:11,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431494 2024-11-24T02:47:11,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=455, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431494) roll requested 2024-11-24T02:47:11,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431503 2024-11-24T02:47:11,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431494 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431503 2024-11-24T02:47:11,513 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,513 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=456, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431503) roll requested 2024-11-24T02:47:11,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431513 2024-11-24T02:47:11,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,523 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,523 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,523 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,523 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431513 2024-11-24T02:47:11,524 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,524 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,524 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=457, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,524 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,524 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,524 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431513) roll requested 2024-11-24T02:47:11,524 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431524 2024-11-24T02:47:11,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431524 2024-11-24T02:47:11,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,531 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=458, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,531 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,531 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431524) roll requested 2024-11-24T02:47:11,531 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431531 2024-11-24T02:47:11,535 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,535 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,535 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,535 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,535 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431524 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431531 2024-11-24T02:47:11,536 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,536 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=459, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431531) roll requested 2024-11-24T02:47:11,537 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431537 2024-11-24T02:47:11,541 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,541 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431531 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431537 2024-11-24T02:47:11,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,544 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=460, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,544 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,544 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,544 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431537) roll requested 2024-11-24T02:47:11,544 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431544 2024-11-24T02:47:11,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,549 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,549 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431537 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431544 2024-11-24T02:47:11,550 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,550 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=461, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431544) roll requested 2024-11-24T02:47:11,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431550 2024-11-24T02:47:11,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,555 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,555 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,555 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431544 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431550 2024-11-24T02:47:11,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=462, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,556 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431550) roll requested 2024-11-24T02:47:11,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431556 2024-11-24T02:47:11,560 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,560 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,560 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,560 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,560 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,560 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431556 2024-11-24T02:47:11,561 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,561 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,561 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,561 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=463, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431556) roll requested 2024-11-24T02:47:11,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431561 2024-11-24T02:47:11,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,566 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,566 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431556 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431561 2024-11-24T02:47:11,567 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,568 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=464, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,568 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,568 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431561) roll requested 2024-11-24T02:47:11,568 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431568 2024-11-24T02:47:11,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431568 2024-11-24T02:47:11,575 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=465, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431568) roll requested 2024-11-24T02:47:11,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431576 2024-11-24T02:47:11,583 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,583 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431568 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431576 2024-11-24T02:47:11,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,585 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=466, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,585 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,585 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,585 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431576) roll requested 2024-11-24T02:47:11,585 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431585 2024-11-24T02:47:11,592 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,592 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,593 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431585 2024-11-24T02:47:11,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,594 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,594 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=467, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,594 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,594 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,594 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431585) roll requested 2024-11-24T02:47:11,594 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431594 2024-11-24T02:47:11,602 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,603 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,603 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431585 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431594 2024-11-24T02:47:11,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,608 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=468, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,608 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,608 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,608 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431594) roll requested 2024-11-24T02:47:11,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431608 2024-11-24T02:47:11,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431594 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431608 2024-11-24T02:47:11,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=469, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431608) roll requested 2024-11-24T02:47:11,614 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431614 2024-11-24T02:47:11,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,618 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431614 2024-11-24T02:47:11,619 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,619 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=470, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431614) roll requested 2024-11-24T02:47:11,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431619 2024-11-24T02:47:11,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,625 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,625 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431619 2024-11-24T02:47:11,626 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,626 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,626 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,627 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=471, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,627 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,627 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431619) roll requested 2024-11-24T02:47:11,627 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431627 2024-11-24T02:47:11,632 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,633 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,633 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,633 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,633 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,633 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431627 2024-11-24T02:47:11,633 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,634 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,634 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,634 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=472, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,634 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,634 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431627) roll requested 2024-11-24T02:47:11,634 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431634 2024-11-24T02:47:11,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,641 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,641 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,641 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,641 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,641 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431627 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431634 2024-11-24T02:47:11,643 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,643 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,643 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=473, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,644 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431634) roll requested 2024-11-24T02:47:11,644 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431644 2024-11-24T02:47:11,648 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,648 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,648 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431634 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431644 2024-11-24T02:47:11,649 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,649 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=474, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431644) roll requested 2024-11-24T02:47:11,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431649 2024-11-24T02:47:11,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431644 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431649 2024-11-24T02:47:11,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,654 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,654 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=475, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,654 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,654 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,654 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431649) roll requested 2024-11-24T02:47:11,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431655 2024-11-24T02:47:11,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,660 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431655 2024-11-24T02:47:11,661 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,661 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,661 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=476, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,661 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,661 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,661 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431655) roll requested 2024-11-24T02:47:11,661 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431661 2024-11-24T02:47:11,666 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,666 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,666 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,666 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,666 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,666 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431661 2024-11-24T02:47:11,667 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,667 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,668 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=477, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,668 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,668 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,668 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431661) roll requested 2024-11-24T02:47:11,668 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431668 2024-11-24T02:47:11,673 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431661 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431668 2024-11-24T02:47:11,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=478, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431668) roll requested 2024-11-24T02:47:11,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431675 2024-11-24T02:47:11,682 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,682 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,682 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,682 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,683 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,683 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431668 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431675 2024-11-24T02:47:11,683 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,683 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,683 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=479, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,683 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,683 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,684 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431675) roll requested 2024-11-24T02:47:11,684 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431684 2024-11-24T02:47:11,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431684 2024-11-24T02:47:11,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=480, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431684) roll requested 2024-11-24T02:47:11,691 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431691 2024-11-24T02:47:11,698 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,699 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,699 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431684 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431691 2024-11-24T02:47:11,699 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,699 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,700 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=481, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,700 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,700 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,700 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431691) roll requested 2024-11-24T02:47:11,700 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431700 2024-11-24T02:47:11,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,708 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,708 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,708 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,708 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431700 2024-11-24T02:47:11,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,709 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,709 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=482, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431700) roll requested 2024-11-24T02:47:11,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431709 2024-11-24T02:47:11,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431700 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431709 2024-11-24T02:47:11,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,716 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,716 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=483, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,716 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,716 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431709) roll requested 2024-11-24T02:47:11,716 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431716 2024-11-24T02:47:11,723 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,723 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431716 2024-11-24T02:47:11,724 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=484, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431716) roll requested 2024-11-24T02:47:11,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431725 2024-11-24T02:47:11,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431716 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431725 2024-11-24T02:47:11,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=485, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,732 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,732 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431725) roll requested 2024-11-24T02:47:11,732 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431732 2024-11-24T02:47:11,736 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,737 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431732 2024-11-24T02:47:11,737 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,737 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,737 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=486, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,738 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,738 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431732) roll requested 2024-11-24T02:47:11,738 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431738 2024-11-24T02:47:11,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,743 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,743 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,743 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431732 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431738 2024-11-24T02:47:11,743 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,743 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=487, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,743 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,744 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,744 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431738) roll requested 2024-11-24T02:47:11,744 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431744 2024-11-24T02:47:11,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431738 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431744 2024-11-24T02:47:11,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=488, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431744) roll requested 2024-11-24T02:47:11,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431751 2024-11-24T02:47:11,757 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,757 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,757 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,757 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431744 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431751 2024-11-24T02:47:11,758 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,758 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=489, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431751) roll requested 2024-11-24T02:47:11,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431759 2024-11-24T02:47:11,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431759 2024-11-24T02:47:11,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=490, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431759) roll requested 2024-11-24T02:47:11,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431767 2024-11-24T02:47:11,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431767 2024-11-24T02:47:11,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=491, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431767) roll requested 2024-11-24T02:47:11,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431776 2024-11-24T02:47:11,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,783 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431776 2024-11-24T02:47:11,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=492, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431776) roll requested 2024-11-24T02:47:11,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431785 2024-11-24T02:47:11,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431785 2024-11-24T02:47:11,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=493, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431785) roll requested 2024-11-24T02:47:11,791 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431791 2024-11-24T02:47:11,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,796 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,796 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,796 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,796 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,796 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431791 2024-11-24T02:47:11,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,797 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,797 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=494, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,797 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,797 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,797 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431791) roll requested 2024-11-24T02:47:11,797 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431797 2024-11-24T02:47:11,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,803 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,803 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431791 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431797 2024-11-24T02:47:11,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=495, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,804 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,804 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431797) roll requested 2024-11-24T02:47:11,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431804 2024-11-24T02:47:11,811 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,811 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,812 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,812 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,812 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,812 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431797 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431804 2024-11-24T02:47:11,813 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,813 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,813 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=496, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,813 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,813 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,813 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431804) roll requested 2024-11-24T02:47:11,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431813 2024-11-24T02:47:11,820 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,821 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,821 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431804 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431813 2024-11-24T02:47:11,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,822 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=497, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,822 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,822 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,822 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431813) roll requested 2024-11-24T02:47:11,822 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431822 2024-11-24T02:47:11,829 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,829 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,829 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,829 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,829 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,829 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431822 2024-11-24T02:47:11,830 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,830 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=498, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431822) roll requested 2024-11-24T02:47:11,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431831 2024-11-24T02:47:11,837 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,837 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,837 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,837 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,837 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,838 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431822 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431831 2024-11-24T02:47:11,838 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,838 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,839 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,839 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=499, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,839 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,839 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431831) roll requested 2024-11-24T02:47:11,839 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431839 2024-11-24T02:47:11,845 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,845 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,846 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,846 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,846 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,846 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431839 2024-11-24T02:47:11,847 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,847 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,847 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,847 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=500, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,847 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,847 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431839) roll requested 2024-11-24T02:47:11,848 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431847 2024-11-24T02:47:11,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431839 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431847 2024-11-24T02:47:11,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=501, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,857 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,857 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431847) roll requested 2024-11-24T02:47:11,857 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431857 2024-11-24T02:47:11,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,865 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,865 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431847 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431857 2024-11-24T02:47:11,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,866 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=502, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431857) roll requested 2024-11-24T02:47:11,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431866 2024-11-24T02:47:11,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431857 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431866 2024-11-24T02:47:11,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=503, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431866) roll requested 2024-11-24T02:47:11,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431874 2024-11-24T02:47:11,881 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,881 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,881 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,881 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,882 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431874 2024-11-24T02:47:11,883 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=504, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431874) roll requested 2024-11-24T02:47:11,884 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431884 2024-11-24T02:47:11,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,892 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431884 2024-11-24T02:47:11,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,893 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=505, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,893 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431884) roll requested 2024-11-24T02:47:11,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431893 2024-11-24T02:47:11,898 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,898 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,898 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,898 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,898 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,898 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431884 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431893 2024-11-24T02:47:11,899 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,899 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,899 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=506, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,899 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,899 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,899 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431893) roll requested 2024-11-24T02:47:11,899 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431899 2024-11-24T02:47:11,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431899 2024-11-24T02:47:11,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=507, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431899) roll requested 2024-11-24T02:47:11,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431906 2024-11-24T02:47:11,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431899 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431906 2024-11-24T02:47:11,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,914 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=508, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431906) roll requested 2024-11-24T02:47:11,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431914 2024-11-24T02:47:11,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,921 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,921 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,921 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431914 2024-11-24T02:47:11,922 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,922 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=509, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431914) roll requested 2024-11-24T02:47:11,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431922 2024-11-24T02:47:11,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431922 2024-11-24T02:47:11,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=510, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431922) roll requested 2024-11-24T02:47:11,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431931 2024-11-24T02:47:11,937 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,937 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,937 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,937 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,937 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,937 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431931 2024-11-24T02:47:11,938 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,939 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=511, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,939 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,939 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431931) roll requested 2024-11-24T02:47:11,939 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431939 2024-11-24T02:47:11,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431939 2024-11-24T02:47:11,945 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,945 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=512, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,945 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,945 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,945 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,945 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431939) roll requested 2024-11-24T02:47:11,946 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431946 2024-11-24T02:47:11,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,950 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,950 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,950 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,950 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431939 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431946 2024-11-24T02:47:11,951 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,951 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,951 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,951 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=513, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,951 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,951 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431946) roll requested 2024-11-24T02:47:11,951 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431951 2024-11-24T02:47:11,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431946 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431951 2024-11-24T02:47:11,958 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,958 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=514, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431951) roll requested 2024-11-24T02:47:11,959 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431959 2024-11-24T02:47:11,964 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,964 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431951 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431959 2024-11-24T02:47:11,965 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=515, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431959) roll requested 2024-11-24T02:47:11,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431966 2024-11-24T02:47:11,972 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,972 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,973 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,973 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,973 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431959 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431966 2024-11-24T02:47:11,974 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,974 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,974 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,974 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=516, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,974 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,974 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431966) roll requested 2024-11-24T02:47:11,974 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431974 2024-11-24T02:47:11,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,983 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,983 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,983 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431974 2024-11-24T02:47:11,984 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,984 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,984 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,984 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=517, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431974) roll requested 2024-11-24T02:47:11,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431985 2024-11-24T02:47:11,994 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,994 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:11,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431974 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431985 2024-11-24T02:47:11,995 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:11,995 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,995 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:11,995 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=518, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:11,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:11,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431985) roll requested 2024-11-24T02:47:11,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416431995 2024-11-24T02:47:12,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,001 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,001 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,001 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431995 2024-11-24T02:47:12,001 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,001 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=519, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416431995) roll requested 2024-11-24T02:47:12,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432002 2024-11-24T02:47:12,008 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,009 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,009 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,009 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,009 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,009 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416431995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432002 2024-11-24T02:47:12,010 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,010 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,010 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,010 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=520, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,010 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,010 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432002) roll requested 2024-11-24T02:47:12,010 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432010 2024-11-24T02:47:12,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432010 2024-11-24T02:47:12,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=521, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432010) roll requested 2024-11-24T02:47:12,019 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432019 2024-11-24T02:47:12,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432010 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432019 2024-11-24T02:47:12,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,027 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=522, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,027 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,027 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432019) roll requested 2024-11-24T02:47:12,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432028 2024-11-24T02:47:12,034 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,034 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,035 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,035 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,035 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432019 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432028 2024-11-24T02:47:12,035 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,035 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,036 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,036 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=523, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,036 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,036 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432028) roll requested 2024-11-24T02:47:12,036 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432036 2024-11-24T02:47:12,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,044 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,044 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,044 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,044 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432036 2024-11-24T02:47:12,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=524, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,045 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,045 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432036) roll requested 2024-11-24T02:47:12,045 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432045 2024-11-24T02:47:12,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432036 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432045 2024-11-24T02:47:12,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=525, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432045) roll requested 2024-11-24T02:47:12,053 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432053 2024-11-24T02:47:12,058 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,058 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,058 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,058 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,058 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,058 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432045 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432053 2024-11-24T02:47:12,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=526, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,061 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432053) roll requested 2024-11-24T02:47:12,061 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432061 2024-11-24T02:47:12,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432053 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432061 2024-11-24T02:47:12,072 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,072 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,072 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=527, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,072 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,073 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,073 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432061) roll requested 2024-11-24T02:47:12,073 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432073 2024-11-24T02:47:12,077 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,078 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,078 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,078 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,078 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,078 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432061 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432073 2024-11-24T02:47:12,078 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,078 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,079 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=528, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,078 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,079 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,079 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432073) roll requested 2024-11-24T02:47:12,079 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432079 2024-11-24T02:47:12,083 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,083 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,083 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,083 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,083 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432073 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432079 2024-11-24T02:47:12,084 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=529, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,085 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,085 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432079) roll requested 2024-11-24T02:47:12,085 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432085 2024-11-24T02:47:12,089 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,089 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,089 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,090 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,090 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,090 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432079 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432085 2024-11-24T02:47:12,090 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,090 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=530, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,091 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432085) roll requested 2024-11-24T02:47:12,091 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432091 2024-11-24T02:47:12,097 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,097 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,097 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,097 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,097 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,097 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432085 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432091 2024-11-24T02:47:12,098 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,098 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,098 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=531, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,098 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,098 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,098 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432091) roll requested 2024-11-24T02:47:12,098 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432098 2024-11-24T02:47:12,103 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,103 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,103 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,103 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,103 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,103 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432091 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432098 2024-11-24T02:47:12,104 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,104 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,104 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=532, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,105 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,105 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432098) roll requested 2024-11-24T02:47:12,105 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432105 2024-11-24T02:47:12,109 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,109 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,109 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,109 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,109 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432098 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432105 2024-11-24T02:47:12,110 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,110 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,110 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,110 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=533, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,110 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432105) roll requested 2024-11-24T02:47:12,111 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432111 2024-11-24T02:47:12,115 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,115 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,115 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,115 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,116 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,116 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432105 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432111 2024-11-24T02:47:12,116 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,116 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,116 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,117 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=534, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,117 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,117 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432111) roll requested 2024-11-24T02:47:12,117 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432117 2024-11-24T02:47:12,121 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,121 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,121 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,121 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,121 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,121 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432111 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432117 2024-11-24T02:47:12,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=535, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,122 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432117) roll requested 2024-11-24T02:47:12,122 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432122 2024-11-24T02:47:12,127 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,127 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432117 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432122 2024-11-24T02:47:12,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=536, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432122) roll requested 2024-11-24T02:47:12,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432128 2024-11-24T02:47:12,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432122 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432128 2024-11-24T02:47:12,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=537, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432128) roll requested 2024-11-24T02:47:12,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432135 2024-11-24T02:47:12,140 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,140 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,140 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,140 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,140 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,140 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432135 2024-11-24T02:47:12,140 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,140 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,141 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=538, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,141 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,141 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,141 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432135) roll requested 2024-11-24T02:47:12,141 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432141 2024-11-24T02:47:12,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,146 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,146 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,146 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,146 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432141 2024-11-24T02:47:12,146 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,146 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=539, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,146 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,146 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,147 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,147 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432141) roll requested 2024-11-24T02:47:12,147 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432147 2024-11-24T02:47:12,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432141 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432147 2024-11-24T02:47:12,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=540, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432147) roll requested 2024-11-24T02:47:12,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432153 2024-11-24T02:47:12,157 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432147 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432153 2024-11-24T02:47:12,158 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,158 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,158 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=541, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,158 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,159 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,159 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432153) roll requested 2024-11-24T02:47:12,159 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432159 2024-11-24T02:47:12,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432159 2024-11-24T02:47:12,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,165 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=542, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,165 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,165 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432159) roll requested 2024-11-24T02:47:12,165 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432165 2024-11-24T02:47:12,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,171 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432159 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432165 2024-11-24T02:47:12,171 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,171 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,172 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=543, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,172 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,172 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432165) roll requested 2024-11-24T02:47:12,172 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432172 2024-11-24T02:47:12,177 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,177 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,177 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,177 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,177 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,177 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432165 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432172 2024-11-24T02:47:12,177 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,178 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,178 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=544, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,178 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,178 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,178 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432172) roll requested 2024-11-24T02:47:12,178 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432178 2024-11-24T02:47:12,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432172 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432178 2024-11-24T02:47:12,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=545, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432178) roll requested 2024-11-24T02:47:12,188 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432188 2024-11-24T02:47:12,192 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432178 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432188 2024-11-24T02:47:12,193 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,193 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,193 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=546, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,193 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,194 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,194 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432188) roll requested 2024-11-24T02:47:12,194 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432194 2024-11-24T02:47:12,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,198 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432188 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432194 2024-11-24T02:47:12,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=547, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,199 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432194) roll requested 2024-11-24T02:47:12,199 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432199 2024-11-24T02:47:12,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,204 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432194 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432199 2024-11-24T02:47:12,204 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,204 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,205 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=548, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,205 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,205 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,205 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432199) roll requested 2024-11-24T02:47:12,205 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432205 2024-11-24T02:47:12,209 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,209 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432199 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432205 2024-11-24T02:47:12,210 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,210 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=549, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,210 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432205) roll requested 2024-11-24T02:47:12,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432211 2024-11-24T02:47:12,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432205 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432211 2024-11-24T02:47:12,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=550, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432211) roll requested 2024-11-24T02:47:12,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432217 2024-11-24T02:47:12,225 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,225 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,225 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,225 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,225 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,225 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432217 2024-11-24T02:47:12,226 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,226 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,226 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=551, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,226 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,226 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,226 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432217) roll requested 2024-11-24T02:47:12,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432226 2024-11-24T02:47:12,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,231 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,231 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,231 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432226 2024-11-24T02:47:12,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=552, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432226) roll requested 2024-11-24T02:47:12,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432232 2024-11-24T02:47:12,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,237 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432226 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432232 2024-11-24T02:47:12,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=553, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432232) roll requested 2024-11-24T02:47:12,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432238 2024-11-24T02:47:12,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432238 2024-11-24T02:47:12,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,244 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=554, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432238) roll requested 2024-11-24T02:47:12,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432245 2024-11-24T02:47:12,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432245 2024-11-24T02:47:12,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=555, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,251 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,251 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432245) roll requested 2024-11-24T02:47:12,251 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432251 2024-11-24T02:47:12,256 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,256 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432251 2024-11-24T02:47:12,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,258 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=556, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,258 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,258 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432251) roll requested 2024-11-24T02:47:12,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432258 2024-11-24T02:47:12,262 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,262 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,262 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,262 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,262 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,262 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432251 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432258 2024-11-24T02:47:12,263 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,263 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,263 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=557, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,263 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,263 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,263 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432258) roll requested 2024-11-24T02:47:12,263 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432263 2024-11-24T02:47:12,267 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,267 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,267 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,268 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,268 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432263 2024-11-24T02:47:12,268 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,268 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=558, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,268 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,269 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,269 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432263) roll requested 2024-11-24T02:47:12,269 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432269 2024-11-24T02:47:12,273 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,273 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,274 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432263 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432269 2024-11-24T02:47:12,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=559, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432269) roll requested 2024-11-24T02:47:12,275 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432274 2024-11-24T02:47:12,278 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432269 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432274 2024-11-24T02:47:12,279 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,279 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,279 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,279 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=560, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,280 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,280 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432274) roll requested 2024-11-24T02:47:12,280 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432280 2024-11-24T02:47:12,284 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,284 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,284 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,284 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,284 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,284 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432280 2024-11-24T02:47:12,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,285 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=561, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,285 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,285 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,285 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432280) roll requested 2024-11-24T02:47:12,285 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432285 2024-11-24T02:47:12,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432280 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432285 2024-11-24T02:47:12,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=562, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,294 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,294 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432285) roll requested 2024-11-24T02:47:12,294 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432294 2024-11-24T02:47:12,298 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,298 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,298 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,298 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,298 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,298 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432285 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432294 2024-11-24T02:47:12,299 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,299 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,299 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=563, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,299 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,299 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,299 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432294) roll requested 2024-11-24T02:47:12,299 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432299 2024-11-24T02:47:12,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,304 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,304 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432294 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432299 2024-11-24T02:47:12,305 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,305 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,305 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=564, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,305 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432299) roll requested 2024-11-24T02:47:12,306 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432305 2024-11-24T02:47:12,310 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,311 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,311 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,311 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,311 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,311 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432299 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432305 2024-11-24T02:47:12,311 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,311 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,312 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,312 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=565, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,312 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,312 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432305) roll requested 2024-11-24T02:47:12,312 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432312 2024-11-24T02:47:12,316 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,316 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,316 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,317 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,317 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,317 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432312 2024-11-24T02:47:12,317 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,318 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=566, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,318 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,318 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432312) roll requested 2024-11-24T02:47:12,318 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432318 2024-11-24T02:47:12,323 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,323 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,323 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,323 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,323 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432312 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432318 2024-11-24T02:47:12,324 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,324 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,324 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,324 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=567, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432318) roll requested 2024-11-24T02:47:12,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432324 2024-11-24T02:47:12,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,394 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,394 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,394 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,394 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,394 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432318 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432324 2024-11-24T02:47:12,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=568, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432324) roll requested 2024-11-24T02:47:12,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432397 2024-11-24T02:47:12,402 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,402 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,402 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,402 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,402 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,402 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432397 2024-11-24T02:47:12,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=569, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432397) roll requested 2024-11-24T02:47:12,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432403 2024-11-24T02:47:12,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432403 2024-11-24T02:47:12,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=570, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432403) roll requested 2024-11-24T02:47:12,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432408 2024-11-24T02:47:12,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432408 2024-11-24T02:47:12,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=571, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432408) roll requested 2024-11-24T02:47:12,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432416 2024-11-24T02:47:12,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432416 2024-11-24T02:47:12,421 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,421 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=572, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432416) roll requested 2024-11-24T02:47:12,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432422 2024-11-24T02:47:12,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,428 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,428 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,428 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432422 2024-11-24T02:47:12,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,429 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=573, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,429 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,429 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432422) roll requested 2024-11-24T02:47:12,429 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432429 2024-11-24T02:47:12,435 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,435 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,435 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,435 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,435 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,435 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432429 2024-11-24T02:47:12,436 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,436 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,436 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=574, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432429) roll requested 2024-11-24T02:47:12,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432436 2024-11-24T02:47:12,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,444 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432429 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432436 2024-11-24T02:47:12,445 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,445 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,445 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=575, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,445 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,445 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,445 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432436) roll requested 2024-11-24T02:47:12,445 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432445 2024-11-24T02:47:12,452 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,452 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,452 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,452 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,453 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,453 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432445 2024-11-24T02:47:12,453 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,453 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,454 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=576, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,454 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,454 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,454 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432445) roll requested 2024-11-24T02:47:12,454 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432454 2024-11-24T02:47:12,460 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,460 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,460 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,460 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,461 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,461 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432445 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432454 2024-11-24T02:47:12,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=577, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,462 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,462 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432454) roll requested 2024-11-24T02:47:12,462 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432462 2024-11-24T02:47:12,468 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,468 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,468 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,468 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,468 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,468 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432454 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432462 2024-11-24T02:47:12,469 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,469 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=578, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432462) roll requested 2024-11-24T02:47:12,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432470 2024-11-24T02:47:12,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,476 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,476 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,476 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,476 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432462 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432470 2024-11-24T02:47:12,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=579, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,477 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,477 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432470) roll requested 2024-11-24T02:47:12,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432477 2024-11-24T02:47:12,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432477 2024-11-24T02:47:12,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,486 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,486 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,486 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=580, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,486 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,486 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432477) roll requested 2024-11-24T02:47:12,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432486 2024-11-24T02:47:12,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432477 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432486 2024-11-24T02:47:12,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=581, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432486) roll requested 2024-11-24T02:47:12,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432497 2024-11-24T02:47:12,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,501 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,501 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,501 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,501 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,501 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432486 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432497 2024-11-24T02:47:12,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,502 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=582, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,502 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,502 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432497) roll requested 2024-11-24T02:47:12,502 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432502 2024-11-24T02:47:12,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432502 2024-11-24T02:47:12,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,507 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=583, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,507 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,507 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432502) roll requested 2024-11-24T02:47:12,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432507 2024-11-24T02:47:12,513 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,513 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432502 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432507 2024-11-24T02:47:12,514 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,514 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,514 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,514 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=584, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,514 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,514 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432507) roll requested 2024-11-24T02:47:12,514 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432514 2024-11-24T02:47:12,520 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,520 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,520 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,520 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,520 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,520 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432514 2024-11-24T02:47:12,521 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,521 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,521 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,521 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=585, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,522 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,522 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432514) roll requested 2024-11-24T02:47:12,522 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432522 2024-11-24T02:47:12,528 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,528 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,529 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,529 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,529 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,529 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432514 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432522 2024-11-24T02:47:12,529 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,530 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,530 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=586, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,530 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,530 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,530 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432522) roll requested 2024-11-24T02:47:12,530 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432530 2024-11-24T02:47:12,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,537 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432522 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432530 2024-11-24T02:47:12,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=587, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,538 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,538 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432530) roll requested 2024-11-24T02:47:12,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432538 2024-11-24T02:47:12,543 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,543 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,543 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,543 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,543 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432530 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432538 2024-11-24T02:47:12,544 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,544 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,545 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=588, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,545 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,545 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432538) roll requested 2024-11-24T02:47:12,545 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432545 2024-11-24T02:47:12,552 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,552 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,553 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,553 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,553 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,553 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432545 2024-11-24T02:47:12,554 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,554 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,554 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,554 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=589, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,554 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,554 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432545) roll requested 2024-11-24T02:47:12,554 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432554 2024-11-24T02:47:12,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432545 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432554 2024-11-24T02:47:12,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=590, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432554) roll requested 2024-11-24T02:47:12,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432563 2024-11-24T02:47:12,571 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,571 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,571 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,571 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,572 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,572 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432554 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432563 2024-11-24T02:47:12,573 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,573 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,573 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,573 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=591, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,573 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,573 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432563) roll requested 2024-11-24T02:47:12,573 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432573 2024-11-24T02:47:12,579 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,579 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,580 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,580 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,580 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432563 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432573 2024-11-24T02:47:12,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=592, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,580 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,580 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432573) roll requested 2024-11-24T02:47:12,580 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432580 2024-11-24T02:47:12,585 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,585 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,585 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,585 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,585 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,585 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432573 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432580 2024-11-24T02:47:12,585 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,585 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,586 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,586 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=593, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,586 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,586 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432580) roll requested 2024-11-24T02:47:12,586 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432586 2024-11-24T02:47:12,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,590 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,590 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432580 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432586 2024-11-24T02:47:12,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,591 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=594, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,591 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,591 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432586) roll requested 2024-11-24T02:47:12,591 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432591 2024-11-24T02:47:12,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,597 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,597 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432586 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432591 2024-11-24T02:47:12,598 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,598 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,598 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,598 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=595, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,598 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,598 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432591) roll requested 2024-11-24T02:47:12,599 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432599 2024-11-24T02:47:12,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,605 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,605 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,605 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,605 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432591 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432599 2024-11-24T02:47:12,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=596, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432599) roll requested 2024-11-24T02:47:12,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432607 2024-11-24T02:47:12,613 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,613 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,613 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,613 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,613 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,613 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432599 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432607 2024-11-24T02:47:12,614 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,614 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,614 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,614 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=597, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,614 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,614 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432607) roll requested 2024-11-24T02:47:12,615 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432614 2024-11-24T02:47:12,620 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,620 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,620 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,620 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,620 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,620 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432614 2024-11-24T02:47:12,621 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,621 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,621 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,622 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=598, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,622 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,622 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432614) roll requested 2024-11-24T02:47:12,622 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432622 2024-11-24T02:47:12,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432614 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432622 2024-11-24T02:47:12,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=599, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432622) roll requested 2024-11-24T02:47:12,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432630 2024-11-24T02:47:12,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,639 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,639 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,639 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,639 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,639 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432622 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432630 2024-11-24T02:47:12,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,640 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,640 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=600, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,640 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,640 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432630) roll requested 2024-11-24T02:47:12,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432640 2024-11-24T02:47:12,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,648 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,648 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,648 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432640 2024-11-24T02:47:12,648 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,648 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,649 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,649 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=601, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,649 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,649 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432640) roll requested 2024-11-24T02:47:12,649 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432649 2024-11-24T02:47:12,654 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432640 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432649 2024-11-24T02:47:12,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=602, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432649) roll requested 2024-11-24T02:47:12,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432655 2024-11-24T02:47:12,660 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,661 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,661 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,661 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432649 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432655 2024-11-24T02:47:12,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=603, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432655) roll requested 2024-11-24T02:47:12,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432664 2024-11-24T02:47:12,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432664 2024-11-24T02:47:12,672 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,672 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=604, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432664) roll requested 2024-11-24T02:47:12,673 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432672 2024-11-24T02:47:12,677 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,677 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432672 2024-11-24T02:47:12,678 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,678 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,678 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,678 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=605, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432672) roll requested 2024-11-24T02:47:12,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432678 2024-11-24T02:47:12,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,684 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,684 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,684 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,684 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432678 2024-11-24T02:47:12,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,685 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,685 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=606, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,685 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,685 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,685 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432678) roll requested 2024-11-24T02:47:12,685 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432685 2024-11-24T02:47:12,689 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,689 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,689 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,689 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432685 2024-11-24T02:47:12,690 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,690 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,690 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=607, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,690 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,690 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,690 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432685) roll requested 2024-11-24T02:47:12,690 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432690 2024-11-24T02:47:12,695 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,695 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,695 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,695 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432685 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432690 2024-11-24T02:47:12,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=608, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432690) roll requested 2024-11-24T02:47:12,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432696 2024-11-24T02:47:12,702 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432690 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432696 2024-11-24T02:47:12,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=609, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432696) roll requested 2024-11-24T02:47:12,704 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432703 2024-11-24T02:47:12,709 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,710 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432703 2024-11-24T02:47:12,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,711 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=610, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,711 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,711 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432703) roll requested 2024-11-24T02:47:12,711 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432711 2024-11-24T02:47:12,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,719 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432711 2024-11-24T02:47:12,720 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,720 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,720 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,721 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=611, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,721 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,721 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432711) roll requested 2024-11-24T02:47:12,721 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432721 2024-11-24T02:47:12,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,728 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,728 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,728 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,728 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432711 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432721 2024-11-24T02:47:12,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,729 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=612, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432721) roll requested 2024-11-24T02:47:12,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432730 2024-11-24T02:47:12,737 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,737 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,737 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,737 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,737 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,738 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432721 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432730 2024-11-24T02:47:12,739 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,739 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,739 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,739 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=613, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,739 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,739 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432730) roll requested 2024-11-24T02:47:12,740 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432739 2024-11-24T02:47:12,745 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,745 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,745 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,745 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,746 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,746 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432739 2024-11-24T02:47:12,748 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,748 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,749 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=614, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,749 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,749 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432739) roll requested 2024-11-24T02:47:12,749 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432749 2024-11-24T02:47:12,753 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,753 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,753 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432739 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432749 2024-11-24T02:47:12,756 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,756 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,756 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=615, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,756 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432749) roll requested 2024-11-24T02:47:12,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432757 2024-11-24T02:47:12,761 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,762 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,762 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,762 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,762 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432749 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432757 2024-11-24T02:47:12,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,764 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=616, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,764 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432757) roll requested 2024-11-24T02:47:12,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432765 2024-11-24T02:47:12,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,769 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432765 2024-11-24T02:47:12,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=617, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432765) roll requested 2024-11-24T02:47:12,770 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432770 2024-11-24T02:47:12,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432770 2024-11-24T02:47:12,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=618, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,776 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432770) roll requested 2024-11-24T02:47:12,776 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432776 2024-11-24T02:47:12,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432776 2024-11-24T02:47:12,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=619, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432776) roll requested 2024-11-24T02:47:12,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432785 2024-11-24T02:47:12,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,790 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,790 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,790 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432776 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432785 2024-11-24T02:47:12,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=620, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432785) roll requested 2024-11-24T02:47:12,793 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432793 2024-11-24T02:47:12,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432793 2024-11-24T02:47:12,798 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,798 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,798 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,798 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=621, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432793) roll requested 2024-11-24T02:47:12,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432799 2024-11-24T02:47:12,803 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,803 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,803 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,803 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432799 2024-11-24T02:47:12,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,804 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=622, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432799) roll requested 2024-11-24T02:47:12,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432805 2024-11-24T02:47:12,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,809 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,809 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432805 2024-11-24T02:47:12,812 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,812 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,812 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,812 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=623, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,812 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,812 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432805) roll requested 2024-11-24T02:47:12,813 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432813 2024-11-24T02:47:12,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432813 2024-11-24T02:47:12,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=624, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432813) roll requested 2024-11-24T02:47:12,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432818 2024-11-24T02:47:12,822 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432813 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432818 2024-11-24T02:47:12,823 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,823 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=625, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,824 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,824 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432818) roll requested 2024-11-24T02:47:12,824 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432824 2024-11-24T02:47:12,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432824 2024-11-24T02:47:12,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=626, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432824) roll requested 2024-11-24T02:47:12,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432831 2024-11-24T02:47:12,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432824 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432831 2024-11-24T02:47:12,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=627, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432831) roll requested 2024-11-24T02:47:12,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432837 2024-11-24T02:47:12,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432837 2024-11-24T02:47:12,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=628, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,843 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,843 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432837) roll requested 2024-11-24T02:47:12,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432843 2024-11-24T02:47:12,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432843 2024-11-24T02:47:12,850 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,850 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=629, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,850 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432843) roll requested 2024-11-24T02:47:12,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432851 2024-11-24T02:47:12,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,855 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,855 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,855 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432843 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432851 2024-11-24T02:47:12,856 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,856 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,856 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,856 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=630, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,856 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,856 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432851) roll requested 2024-11-24T02:47:12,856 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432856 2024-11-24T02:47:12,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,861 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,861 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432856 2024-11-24T02:47:12,861 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,861 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,861 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,861 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=631, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,862 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,862 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432856) roll requested 2024-11-24T02:47:12,862 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432862 2024-11-24T02:47:12,866 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,866 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,866 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,866 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432856 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432862 2024-11-24T02:47:12,868 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,868 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,868 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=632, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,868 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,869 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,869 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432862) roll requested 2024-11-24T02:47:12,869 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432869 2024-11-24T02:47:12,873 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,873 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,873 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,873 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,873 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,873 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432862 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432869 2024-11-24T02:47:12,874 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,874 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=633, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,874 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432869) roll requested 2024-11-24T02:47:12,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432874 2024-11-24T02:47:12,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432869 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432874 2024-11-24T02:47:12,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=634, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432874) roll requested 2024-11-24T02:47:12,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432881 2024-11-24T02:47:12,885 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432881 2024-11-24T02:47:12,886 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,886 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,887 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=635, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,887 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,887 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432881) roll requested 2024-11-24T02:47:12,887 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432887 2024-11-24T02:47:12,891 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,891 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,891 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,892 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,892 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,892 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432887 2024-11-24T02:47:12,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=636, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,893 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,893 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432887) roll requested 2024-11-24T02:47:12,893 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432893 2024-11-24T02:47:12,897 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,897 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,897 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,897 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,897 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,897 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432887 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432893 2024-11-24T02:47:12,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,900 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,900 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,900 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=637, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,900 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,900 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432893) roll requested 2024-11-24T02:47:12,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432901 2024-11-24T02:47:12,905 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,905 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,905 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,906 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432893 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432901 2024-11-24T02:47:12,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=638, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,907 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,907 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432901) roll requested 2024-11-24T02:47:12,907 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432907 2024-11-24T02:47:12,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,912 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432907 2024-11-24T02:47:12,912 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,912 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=639, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432907) roll requested 2024-11-24T02:47:12,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432913 2024-11-24T02:47:12,919 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,919 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,919 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,919 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,919 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,919 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432907 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432913 2024-11-24T02:47:12,920 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,920 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,920 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=640, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,920 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432913) roll requested 2024-11-24T02:47:12,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432920 2024-11-24T02:47:12,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432920 2024-11-24T02:47:12,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,926 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=641, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,926 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,926 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432920) roll requested 2024-11-24T02:47:12,926 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432926 2024-11-24T02:47:12,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,931 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432926 2024-11-24T02:47:12,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=642, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432926) roll requested 2024-11-24T02:47:12,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432932 2024-11-24T02:47:12,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432926 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432932 2024-11-24T02:47:12,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=643, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432932) roll requested 2024-11-24T02:47:12,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432937 2024-11-24T02:47:12,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432932 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432937 2024-11-24T02:47:12,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=644, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432937) roll requested 2024-11-24T02:47:12,943 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432943 2024-11-24T02:47:12,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,947 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432943 2024-11-24T02:47:12,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=645, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432943) roll requested 2024-11-24T02:47:12,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432949 2024-11-24T02:47:12,953 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,953 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432949 2024-11-24T02:47:12,954 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,954 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,954 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,954 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=646, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,954 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,954 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432949) roll requested 2024-11-24T02:47:12,954 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432954 2024-11-24T02:47:12,959 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,959 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,959 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,959 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,959 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,959 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432954 2024-11-24T02:47:12,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=647, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432954) roll requested 2024-11-24T02:47:12,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432960 2024-11-24T02:47:12,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,965 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,965 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,965 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432954 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432960 2024-11-24T02:47:12,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,966 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,966 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=648, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,966 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,966 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432960) roll requested 2024-11-24T02:47:12,966 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432966 2024-11-24T02:47:12,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432966 2024-11-24T02:47:12,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=649, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432966) roll requested 2024-11-24T02:47:12,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432972 2024-11-24T02:47:12,976 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,976 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,976 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,976 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432966 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432972 2024-11-24T02:47:12,977 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,977 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,977 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,978 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=650, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,978 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,978 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432972) roll requested 2024-11-24T02:47:12,978 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432978 2024-11-24T02:47:12,982 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,982 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,982 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,982 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,982 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,982 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432978 2024-11-24T02:47:12,983 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,983 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,983 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,983 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=651, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,983 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,983 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432978) roll requested 2024-11-24T02:47:12,983 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432983 2024-11-24T02:47:12,987 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,987 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,987 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,988 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,988 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,988 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432978 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432983 2024-11-24T02:47:12,988 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,988 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,988 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=652, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,988 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,989 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,989 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432983) roll requested 2024-11-24T02:47:12,989 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432989 2024-11-24T02:47:12,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,994 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,994 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,994 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,994 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432983 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432989 2024-11-24T02:47:12,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:12,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:12,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=653, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:12,995 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:12,995 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432989) roll requested 2024-11-24T02:47:12,995 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416432995 2024-11-24T02:47:12,999 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,999 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,999 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,999 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,999 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:12,999 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432989 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432995 2024-11-24T02:47:13,000 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,000 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,000 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,000 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=654, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,000 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,000 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416432995) roll requested 2024-11-24T02:47:13,000 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433000 2024-11-24T02:47:13,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,005 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416432995 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433000 2024-11-24T02:47:13,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=655, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433000) roll requested 2024-11-24T02:47:13,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433005 2024-11-24T02:47:13,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433000 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433005 2024-11-24T02:47:13,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,011 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=656, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,011 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,011 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433005) roll requested 2024-11-24T02:47:13,011 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433011 2024-11-24T02:47:13,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,016 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,016 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,016 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,016 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433011 2024-11-24T02:47:13,017 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,017 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,017 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=657, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,017 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,017 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,017 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433011) roll requested 2024-11-24T02:47:13,017 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433017 2024-11-24T02:47:13,021 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,021 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433011 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433017 2024-11-24T02:47:13,022 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,022 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,022 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,022 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=658, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,023 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433017) roll requested 2024-11-24T02:47:13,023 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433023 2024-11-24T02:47:13,027 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,027 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,027 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,027 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,027 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,027 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433017 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433023 2024-11-24T02:47:13,028 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,028 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,028 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=659, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433023) roll requested 2024-11-24T02:47:13,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433028 2024-11-24T02:47:13,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,034 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433023 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433028 2024-11-24T02:47:13,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,034 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=660, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433028) roll requested 2024-11-24T02:47:13,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433035 2024-11-24T02:47:13,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433035 2024-11-24T02:47:13,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=661, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433035) roll requested 2024-11-24T02:47:13,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433040 2024-11-24T02:47:13,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433040 2024-11-24T02:47:13,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,046 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=662, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,046 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,046 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433040) roll requested 2024-11-24T02:47:13,046 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433046 2024-11-24T02:47:13,051 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,051 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,051 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,051 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,051 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,051 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433046 2024-11-24T02:47:13,052 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,052 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,052 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=663, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,052 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433046) roll requested 2024-11-24T02:47:13,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433052 2024-11-24T02:47:13,056 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,056 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,056 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,056 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,056 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,056 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433046 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433052 2024-11-24T02:47:13,057 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,057 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,057 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,057 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=664, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,057 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433052) roll requested 2024-11-24T02:47:13,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433057 2024-11-24T02:47:13,062 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,062 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,062 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,062 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,062 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,062 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433057 2024-11-24T02:47:13,062 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,063 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,063 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,063 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=665, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,063 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,063 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433057) roll requested 2024-11-24T02:47:13,063 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433063 2024-11-24T02:47:13,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,067 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,067 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,067 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,067 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433063 2024-11-24T02:47:13,068 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,068 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,068 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,068 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=666, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,068 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,068 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433063) roll requested 2024-11-24T02:47:13,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433068 2024-11-24T02:47:13,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433063 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433068 2024-11-24T02:47:13,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=667, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433068) roll requested 2024-11-24T02:47:13,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433075 2024-11-24T02:47:13,081 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,081 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,081 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,081 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,081 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,081 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433075 2024-11-24T02:47:13,082 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,082 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=668, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433075) roll requested 2024-11-24T02:47:13,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433082 2024-11-24T02:47:13,088 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,088 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,088 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,089 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,089 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,089 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433082 2024-11-24T02:47:13,089 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,089 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,090 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,090 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=669, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,090 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,090 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433082) roll requested 2024-11-24T02:47:13,090 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433090 2024-11-24T02:47:13,098 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,098 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,098 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,098 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,098 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,098 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433090 2024-11-24T02:47:13,099 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,099 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,099 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,099 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=670, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,100 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,100 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433090) roll requested 2024-11-24T02:47:13,100 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433100 2024-11-24T02:47:13,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433090 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433100 2024-11-24T02:47:13,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=671, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433100) roll requested 2024-11-24T02:47:13,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433109 2024-11-24T02:47:13,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,117 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,117 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433100 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433109 2024-11-24T02:47:13,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=672, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433109) roll requested 2024-11-24T02:47:13,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433121 2024-11-24T02:47:13,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,126 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433121 2024-11-24T02:47:13,128 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=673, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,129 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433121) roll requested 2024-11-24T02:47:13,129 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433129 2024-11-24T02:47:13,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,136 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,136 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,137 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433129 2024-11-24T02:47:13,137 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,137 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,137 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,137 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=674, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,138 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,138 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433129) roll requested 2024-11-24T02:47:13,138 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433138 2024-11-24T02:47:13,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,142 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433129 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433138 2024-11-24T02:47:13,143 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,143 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,143 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,143 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=675, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,143 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,143 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433138) roll requested 2024-11-24T02:47:13,143 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433143 2024-11-24T02:47:13,147 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,147 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,148 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,148 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,148 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433138 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433143 2024-11-24T02:47:13,148 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,148 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,148 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,148 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=676, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,149 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,149 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433143) roll requested 2024-11-24T02:47:13,149 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433149 2024-11-24T02:47:13,153 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,153 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,153 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,153 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,153 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,153 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433143 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433149 2024-11-24T02:47:13,154 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,154 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,154 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,154 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=677, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433149) roll requested 2024-11-24T02:47:13,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433154 2024-11-24T02:47:13,160 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,160 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,160 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,160 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,160 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,160 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433149 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433154 2024-11-24T02:47:13,161 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,161 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,161 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=678, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433154) roll requested 2024-11-24T02:47:13,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433161 2024-11-24T02:47:13,167 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,167 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,167 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,167 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,167 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,167 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433161 2024-11-24T02:47:13,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,168 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,168 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=679, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,168 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,168 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433161) roll requested 2024-11-24T02:47:13,168 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433168 2024-11-24T02:47:13,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,175 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,175 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,175 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,175 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433168 2024-11-24T02:47:13,176 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,176 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,176 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,176 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=680, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,176 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,177 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433168) roll requested 2024-11-24T02:47:13,177 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433177 2024-11-24T02:47:13,184 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,184 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,184 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,184 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,184 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,184 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433168 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433177 2024-11-24T02:47:13,185 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,185 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,185 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,185 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=681, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,185 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,185 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433177) roll requested 2024-11-24T02:47:13,185 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433185 2024-11-24T02:47:13,191 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,191 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,191 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,191 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,191 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,191 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433177 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433185 2024-11-24T02:47:13,192 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,192 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,192 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=682, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433185) roll requested 2024-11-24T02:47:13,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433192 2024-11-24T02:47:13,198 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,198 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,198 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,198 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,198 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,199 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433185 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433192 2024-11-24T02:47:13,199 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,199 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,199 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,199 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=683, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,199 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,200 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433192) roll requested 2024-11-24T02:47:13,200 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433200 2024-11-24T02:47:13,206 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,206 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,206 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,206 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,207 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,207 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433200 2024-11-24T02:47:13,208 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,208 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,208 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,208 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=684, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,208 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,208 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433200) roll requested 2024-11-24T02:47:13,208 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433208 2024-11-24T02:47:13,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433200 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433208 2024-11-24T02:47:13,217 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,217 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,217 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=685, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433208) roll requested 2024-11-24T02:47:13,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433217 2024-11-24T02:47:13,223 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,223 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,223 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,223 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,223 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,223 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433208 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433217 2024-11-24T02:47:13,224 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,224 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,224 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,224 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=686, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,224 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,224 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433217) roll requested 2024-11-24T02:47:13,224 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433224 2024-11-24T02:47:13,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,230 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433224 2024-11-24T02:47:13,231 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,231 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,231 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=687, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433224) roll requested 2024-11-24T02:47:13,232 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433232 2024-11-24T02:47:13,239 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,239 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,239 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,239 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,239 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,239 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433224 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433232 2024-11-24T02:47:13,240 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,240 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,240 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,240 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=688, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,240 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,240 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433232) roll requested 2024-11-24T02:47:13,241 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433240 2024-11-24T02:47:13,248 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,248 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,248 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433232 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433240 2024-11-24T02:47:13,249 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,249 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,250 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,250 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=689, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,250 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,250 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433240) roll requested 2024-11-24T02:47:13,250 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433250 2024-11-24T02:47:13,255 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,256 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,256 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,256 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433240 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433250 2024-11-24T02:47:13,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=690, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,257 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433250) roll requested 2024-11-24T02:47:13,257 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433257 2024-11-24T02:47:13,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,264 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,264 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433250 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433257 2024-11-24T02:47:13,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,265 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,265 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=691, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433257) roll requested 2024-11-24T02:47:13,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433265 2024-11-24T02:47:13,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,273 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433257 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433265 2024-11-24T02:47:13,274 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,274 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,274 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,274 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=692, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433265) roll requested 2024-11-24T02:47:13,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433274 2024-11-24T02:47:13,280 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,280 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433274 2024-11-24T02:47:13,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=693, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433274) roll requested 2024-11-24T02:47:13,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433282 2024-11-24T02:47:13,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,288 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,288 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,288 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433282 2024-11-24T02:47:13,288 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,288 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,288 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=694, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,289 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,289 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433282) roll requested 2024-11-24T02:47:13,289 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433289 2024-11-24T02:47:13,296 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,296 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,296 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,296 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,296 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,296 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433289 2024-11-24T02:47:13,297 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,297 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,297 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,297 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=695, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,297 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433289) roll requested 2024-11-24T02:47:13,298 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433297 2024-11-24T02:47:13,304 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,304 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,304 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,305 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,305 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,305 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433289 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433297 2024-11-24T02:47:13,307 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=696, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433297) roll requested 2024-11-24T02:47:13,308 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433308 2024-11-24T02:47:13,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433308 2024-11-24T02:47:13,316 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,316 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,316 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,317 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=697, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,317 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,317 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433308) roll requested 2024-11-24T02:47:13,317 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433317 2024-11-24T02:47:13,322 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,322 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,322 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,322 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,322 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,323 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433308 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433317 2024-11-24T02:47:13,323 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,323 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,323 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,323 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=698, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,324 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,324 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433317) roll requested 2024-11-24T02:47:13,324 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433324 2024-11-24T02:47:13,328 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,328 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,328 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,328 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,328 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,328 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433317 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433324 2024-11-24T02:47:13,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=699, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433324) roll requested 2024-11-24T02:47:13,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433333 2024-11-24T02:47:13,337 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,337 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,338 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,338 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,338 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,338 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433324 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433333 2024-11-24T02:47:13,339 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,339 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,339 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,339 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=700, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,339 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,339 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433333) roll requested 2024-11-24T02:47:13,339 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433339 2024-11-24T02:47:13,343 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,344 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,344 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,344 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,344 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,344 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433339 2024-11-24T02:47:13,344 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,344 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,344 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=701, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433339) roll requested 2024-11-24T02:47:13,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433345 2024-11-24T02:47:13,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,349 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,349 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433339 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433345 2024-11-24T02:47:13,350 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,350 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,350 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=702, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433345) roll requested 2024-11-24T02:47:13,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433350 2024-11-24T02:47:13,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,355 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,355 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433350 2024-11-24T02:47:13,356 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,356 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,356 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,356 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=703, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,356 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,356 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433350) roll requested 2024-11-24T02:47:13,357 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433357 2024-11-24T02:47:13,361 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,361 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,361 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,361 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,361 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,361 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433357 2024-11-24T02:47:13,362 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,362 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,362 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,362 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=704, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,362 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,362 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433357) roll requested 2024-11-24T02:47:13,363 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433362 2024-11-24T02:47:13,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433357 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433362 2024-11-24T02:47:13,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=705, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433362) roll requested 2024-11-24T02:47:13,367 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433367 2024-11-24T02:47:13,371 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,371 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,372 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,372 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,372 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,372 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433362 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433367 2024-11-24T02:47:13,372 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,372 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,372 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,373 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=706, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,373 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,373 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433367) roll requested 2024-11-24T02:47:13,373 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433373 2024-11-24T02:47:13,380 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,380 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,380 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,380 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,380 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433367 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433373 2024-11-24T02:47:13,381 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,381 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,382 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,382 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=707, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,382 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,382 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433373) roll requested 2024-11-24T02:47:13,382 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433382 2024-11-24T02:47:13,386 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,386 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,387 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,387 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,387 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,387 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433373 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433382 2024-11-24T02:47:13,387 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,388 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,388 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,388 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=708, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,388 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,388 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433382) roll requested 2024-11-24T02:47:13,388 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433388 2024-11-24T02:47:13,395 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,395 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,395 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,395 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,395 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,396 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433382 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433388 2024-11-24T02:47:13,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=709, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,396 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,396 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433388) roll requested 2024-11-24T02:47:13,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433397 2024-11-24T02:47:13,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433388 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433397 2024-11-24T02:47:13,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=710, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433397) roll requested 2024-11-24T02:47:13,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433403 2024-11-24T02:47:13,407 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433403 2024-11-24T02:47:13,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=711, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433403) roll requested 2024-11-24T02:47:13,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433408 2024-11-24T02:47:13,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433408 2024-11-24T02:47:13,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,415 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=712, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,415 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,415 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433408) roll requested 2024-11-24T02:47:13,415 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433415 2024-11-24T02:47:13,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,421 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,421 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,421 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433415 2024-11-24T02:47:13,422 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,422 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,422 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,422 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=713, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,422 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,422 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433415) roll requested 2024-11-24T02:47:13,422 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433422 2024-11-24T02:47:13,429 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,429 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,430 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,430 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,430 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,430 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433415 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433422 2024-11-24T02:47:13,430 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,431 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,431 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,431 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=714, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,431 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,431 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433422) roll requested 2024-11-24T02:47:13,431 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433431 2024-11-24T02:47:13,437 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,437 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,438 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433422 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433431 2024-11-24T02:47:13,439 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,439 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,439 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,439 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=715, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,439 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,439 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433431) roll requested 2024-11-24T02:47:13,439 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433439 2024-11-24T02:47:13,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,448 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433431 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433439 2024-11-24T02:47:13,449 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,449 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,449 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,450 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=716, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,450 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,450 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433439) roll requested 2024-11-24T02:47:13,450 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433450 2024-11-24T02:47:13,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,456 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,456 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,456 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,456 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433439 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433450 2024-11-24T02:47:13,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,457 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=717, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,457 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,457 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433450) roll requested 2024-11-24T02:47:13,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433457 2024-11-24T02:47:13,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,464 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,464 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433450 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433457 2024-11-24T02:47:13,465 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,465 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,465 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,465 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=718, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,465 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,465 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433457) roll requested 2024-11-24T02:47:13,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433465 2024-11-24T02:47:13,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,471 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,471 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,471 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,471 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,471 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433457 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433465 2024-11-24T02:47:13,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,472 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=719, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,472 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433465) roll requested 2024-11-24T02:47:13,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433472 2024-11-24T02:47:13,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433472 2024-11-24T02:47:13,477 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,477 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,477 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,477 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=720, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,478 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,478 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433472) roll requested 2024-11-24T02:47:13,478 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433478 2024-11-24T02:47:13,482 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,482 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,482 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,482 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433478 2024-11-24T02:47:13,483 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,483 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,483 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=721, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433478) roll requested 2024-11-24T02:47:13,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433483 2024-11-24T02:47:13,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433478 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433483 2024-11-24T02:47:13,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=722, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433483) roll requested 2024-11-24T02:47:13,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433498 2024-11-24T02:47:13,502 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433498 2024-11-24T02:47:13,503 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,503 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,503 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=723, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433498) roll requested 2024-11-24T02:47:13,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433503 2024-11-24T02:47:13,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,507 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433503 2024-11-24T02:47:13,507 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,507 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,507 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,508 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=724, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,508 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,508 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433503) roll requested 2024-11-24T02:47:13,508 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433508 2024-11-24T02:47:13,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,512 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,512 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433508 2024-11-24T02:47:13,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,512 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,512 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=725, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,512 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,512 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433508) roll requested 2024-11-24T02:47:13,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433512 2024-11-24T02:47:13,518 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,518 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,518 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,518 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,518 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433508 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433512 2024-11-24T02:47:13,519 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,519 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,519 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,519 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=726, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433512) roll requested 2024-11-24T02:47:13,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433520 2024-11-24T02:47:13,526 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,526 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,526 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,526 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,526 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,526 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433512 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433520 2024-11-24T02:47:13,526 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,526 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,527 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,527 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=727, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,527 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,527 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433520) roll requested 2024-11-24T02:47:13,527 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433527 2024-11-24T02:47:13,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433527 2024-11-24T02:47:13,533 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=728, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433527) roll requested 2024-11-24T02:47:13,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433533 2024-11-24T02:47:13,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433527 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433533 2024-11-24T02:47:13,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,540 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=729, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,540 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,540 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433533) roll requested 2024-11-24T02:47:13,540 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433540 2024-11-24T02:47:13,546 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,546 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,546 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,546 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433540 2024-11-24T02:47:13,547 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,547 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,547 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,547 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=730, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,547 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,547 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433540) roll requested 2024-11-24T02:47:13,548 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433548 2024-11-24T02:47:13,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433540 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433548 2024-11-24T02:47:13,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=731, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433548) roll requested 2024-11-24T02:47:13,556 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433555 2024-11-24T02:47:13,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,562 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433548 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433555 2024-11-24T02:47:13,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=732, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,563 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,563 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433555) roll requested 2024-11-24T02:47:13,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433564 2024-11-24T02:47:13,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433564 2024-11-24T02:47:13,569 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,569 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=733, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433564) roll requested 2024-11-24T02:47:13,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433569 2024-11-24T02:47:13,574 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,574 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433569 2024-11-24T02:47:13,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=734, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433569) roll requested 2024-11-24T02:47:13,576 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433576 2024-11-24T02:47:13,582 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,582 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,583 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,583 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,583 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,583 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433576 2024-11-24T02:47:13,584 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,584 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,584 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,584 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=735, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,584 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,584 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433576) roll requested 2024-11-24T02:47:13,584 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433584 2024-11-24T02:47:13,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,590 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433576 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433584 2024-11-24T02:47:13,592 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,592 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,592 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=736, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433584) roll requested 2024-11-24T02:47:13,593 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433593 2024-11-24T02:47:13,600 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,600 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,600 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,600 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,601 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,601 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433584 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433593 2024-11-24T02:47:13,601 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,601 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,602 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,602 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=737, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,602 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,602 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433593) roll requested 2024-11-24T02:47:13,602 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433602 2024-11-24T02:47:13,609 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,609 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,609 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,609 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,609 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,609 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433593 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433602 2024-11-24T02:47:13,610 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,610 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,610 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,610 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=738, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,610 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,610 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433602) roll requested 2024-11-24T02:47:13,610 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433610 2024-11-24T02:47:13,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433602 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433610 2024-11-24T02:47:13,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,619 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=739, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433610) roll requested 2024-11-24T02:47:13,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433619 2024-11-24T02:47:13,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433610 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433619 2024-11-24T02:47:13,628 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=740, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433619) roll requested 2024-11-24T02:47:13,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433628 2024-11-24T02:47:13,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433628 2024-11-24T02:47:13,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=741, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433628) roll requested 2024-11-24T02:47:13,636 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433635 2024-11-24T02:47:13,640 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433635 2024-11-24T02:47:13,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,640 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=742, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433635) roll requested 2024-11-24T02:47:13,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433641 2024-11-24T02:47:13,646 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433641 2024-11-24T02:47:13,647 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,647 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,648 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,648 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=743, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,648 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,648 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433641) roll requested 2024-11-24T02:47:13,648 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433648 2024-11-24T02:47:13,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,654 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,654 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,654 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433648 2024-11-24T02:47:13,655 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=744, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433648) roll requested 2024-11-24T02:47:13,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433655 2024-11-24T02:47:13,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433648 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433655 2024-11-24T02:47:13,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=745, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433655) roll requested 2024-11-24T02:47:13,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433663 2024-11-24T02:47:13,670 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,670 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,670 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,670 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,670 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,671 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433663 2024-11-24T02:47:13,671 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,671 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,672 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,672 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=746, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,672 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,672 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433663) roll requested 2024-11-24T02:47:13,672 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433672 2024-11-24T02:47:13,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433672 2024-11-24T02:47:13,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=747, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433672) roll requested 2024-11-24T02:47:13,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433681 2024-11-24T02:47:13,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433672 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433681 2024-11-24T02:47:13,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=748, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433681) roll requested 2024-11-24T02:47:13,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433687 2024-11-24T02:47:13,693 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,693 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,693 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,693 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,693 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433687 2024-11-24T02:47:13,694 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,694 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,694 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,694 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=749, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,695 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,695 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433687) roll requested 2024-11-24T02:47:13,695 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433695 2024-11-24T02:47:13,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,702 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,702 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,702 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,702 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,702 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433695 2024-11-24T02:47:13,703 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,703 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,703 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,703 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=750, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,703 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,703 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433695) roll requested 2024-11-24T02:47:13,703 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433703 2024-11-24T02:47:13,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,711 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,711 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433695 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433703 2024-11-24T02:47:13,712 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,712 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,712 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=751, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433703) roll requested 2024-11-24T02:47:13,713 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433712 2024-11-24T02:47:13,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,720 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,720 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433703 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433712 2024-11-24T02:47:13,721 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,721 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,721 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,722 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=752, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,722 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,722 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433712) roll requested 2024-11-24T02:47:13,722 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433722 2024-11-24T02:47:13,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,731 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433722 2024-11-24T02:47:13,732 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,732 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,732 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=753, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,732 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,733 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,733 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433722) roll requested 2024-11-24T02:47:13,733 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433733 2024-11-24T02:47:13,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,742 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433722 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433733 2024-11-24T02:47:13,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,745 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=754, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433733) roll requested 2024-11-24T02:47:13,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433745 2024-11-24T02:47:13,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433733 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433745 2024-11-24T02:47:13,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=755, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433745) roll requested 2024-11-24T02:47:13,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433751 2024-11-24T02:47:13,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433751 2024-11-24T02:47:13,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=756, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433751) roll requested 2024-11-24T02:47:13,760 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433760 2024-11-24T02:47:13,766 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,767 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,767 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,767 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,767 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433760 2024-11-24T02:47:13,767 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,767 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,767 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=757, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,768 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,768 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433760) roll requested 2024-11-24T02:47:13,768 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433768 2024-11-24T02:47:13,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433760 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433768 2024-11-24T02:47:13,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=758, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433768) roll requested 2024-11-24T02:47:13,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433773 2024-11-24T02:47:13,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433768 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433773 2024-11-24T02:47:13,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=759, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433773) roll requested 2024-11-24T02:47:13,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433779 2024-11-24T02:47:13,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,784 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,784 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,784 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433779 2024-11-24T02:47:13,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=760, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433779) roll requested 2024-11-24T02:47:13,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433785 2024-11-24T02:47:13,789 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,789 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,789 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,789 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,789 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,789 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433785 2024-11-24T02:47:13,790 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,790 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,790 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,790 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=761, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,790 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,790 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433785) roll requested 2024-11-24T02:47:13,790 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433790 2024-11-24T02:47:13,795 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,795 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,795 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,795 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,795 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,795 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433790 2024-11-24T02:47:13,796 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,796 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,796 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,796 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=762, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,796 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,796 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433790) roll requested 2024-11-24T02:47:13,796 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433796 2024-11-24T02:47:13,802 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,802 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433790 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433796 2024-11-24T02:47:13,803 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,803 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,803 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,803 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=763, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,803 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,803 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433796) roll requested 2024-11-24T02:47:13,803 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433803 2024-11-24T02:47:13,809 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,809 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,809 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433796 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433803 2024-11-24T02:47:13,810 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,810 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=764, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433803) roll requested 2024-11-24T02:47:13,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433811 2024-11-24T02:47:13,818 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,818 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,819 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,819 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,819 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,819 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433803 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433811 2024-11-24T02:47:13,820 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,820 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,820 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,820 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=765, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,820 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,820 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433811) roll requested 2024-11-24T02:47:13,820 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433820 2024-11-24T02:47:13,827 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,827 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,827 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,827 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,827 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,827 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433820 2024-11-24T02:47:13,828 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,828 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,828 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,828 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=766, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,828 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,828 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433820) roll requested 2024-11-24T02:47:13,828 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433828 2024-11-24T02:47:13,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,834 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,834 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,834 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433820 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433828 2024-11-24T02:47:13,834 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,834 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,835 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,835 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=767, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,835 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,835 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433828) roll requested 2024-11-24T02:47:13,835 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433835 2024-11-24T02:47:13,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433828 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433835 2024-11-24T02:47:13,843 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,843 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,843 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,843 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=768, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,844 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,844 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433835) roll requested 2024-11-24T02:47:13,844 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433844 2024-11-24T02:47:13,850 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433835 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433844 2024-11-24T02:47:13,851 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,851 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,851 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,851 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=769, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,851 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,851 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433844) roll requested 2024-11-24T02:47:13,851 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433851 2024-11-24T02:47:13,856 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433844 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433851 2024-11-24T02:47:13,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=770, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433851) roll requested 2024-11-24T02:47:13,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433858 2024-11-24T02:47:13,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433851 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433858 2024-11-24T02:47:13,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=771, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433858) roll requested 2024-11-24T02:47:13,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433865 2024-11-24T02:47:13,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433865 2024-11-24T02:47:13,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=772, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433865) roll requested 2024-11-24T02:47:13,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433873 2024-11-24T02:47:13,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,879 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,879 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,879 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,879 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433873 2024-11-24T02:47:13,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=773, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433873) roll requested 2024-11-24T02:47:13,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433880 2024-11-24T02:47:13,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,886 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,886 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433880 2024-11-24T02:47:13,887 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,887 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,887 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=774, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433880) roll requested 2024-11-24T02:47:13,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433888 2024-11-24T02:47:13,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433888 2024-11-24T02:47:13,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=775, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,896 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433888) roll requested 2024-11-24T02:47:13,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433896 2024-11-24T02:47:13,903 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433896 2024-11-24T02:47:13,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=776, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433896) roll requested 2024-11-24T02:47:13,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433904 2024-11-24T02:47:13,911 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,911 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,911 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,911 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,911 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,911 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433904 2024-11-24T02:47:13,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,913 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,913 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=777, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,913 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,913 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433904) roll requested 2024-11-24T02:47:13,913 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433913 2024-11-24T02:47:13,920 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,920 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,920 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,920 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,920 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,920 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433913 2024-11-24T02:47:13,921 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,921 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,922 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,922 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=778, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,922 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,922 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433913) roll requested 2024-11-24T02:47:13,922 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433922 2024-11-24T02:47:13,929 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,929 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,929 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433913 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433922 2024-11-24T02:47:13,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=779, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433922) roll requested 2024-11-24T02:47:13,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433931 2024-11-24T02:47:13,940 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,940 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,940 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,940 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,940 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,940 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433922 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433931 2024-11-24T02:47:13,942 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,942 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,942 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,942 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=780, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,942 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,942 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433931) roll requested 2024-11-24T02:47:13,942 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433942 2024-11-24T02:47:13,951 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,951 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,951 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433942 2024-11-24T02:47:13,952 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,952 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,952 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,952 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=781, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,952 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,952 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433942) roll requested 2024-11-24T02:47:13,952 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433952 2024-11-24T02:47:13,957 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,957 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,957 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,957 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,957 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,957 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433942 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433952 2024-11-24T02:47:13,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,958 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,958 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=782, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,958 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,958 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433952) roll requested 2024-11-24T02:47:13,958 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433958 2024-11-24T02:47:13,963 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,963 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,963 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,963 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,963 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,963 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433952 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433958 2024-11-24T02:47:13,964 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,964 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,964 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=783, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433958) roll requested 2024-11-24T02:47:13,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433964 2024-11-24T02:47:13,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,971 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,971 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433958 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433964 2024-11-24T02:47:13,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,972 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,972 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,972 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=784, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,972 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,972 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433964) roll requested 2024-11-24T02:47:13,972 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433972 2024-11-24T02:47:13,979 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,979 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,979 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,979 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,980 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,980 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433972 2024-11-24T02:47:13,981 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,981 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,981 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,981 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=785, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,982 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,982 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433972) roll requested 2024-11-24T02:47:13,982 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433982 2024-11-24T02:47:13,989 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:13,990 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433972 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433982 2024-11-24T02:47:13,991 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:13,991 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,991 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:13,992 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=786, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:13,992 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:13,992 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433982) roll requested 2024-11-24T02:47:13,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416433992 2024-11-24T02:47:14,001 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,001 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,001 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,002 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,002 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,002 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433982 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433992 2024-11-24T02:47:14,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,003 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=787, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,003 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,003 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416433992) roll requested 2024-11-24T02:47:14,003 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434003 2024-11-24T02:47:14,007 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,007 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,008 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,008 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,008 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,008 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416433992 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434003 2024-11-24T02:47:14,008 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,009 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,009 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,009 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=788, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,009 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,009 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434003) roll requested 2024-11-24T02:47:14,009 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434009 2024-11-24T02:47:14,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434003 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434009 2024-11-24T02:47:14,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,014 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,014 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=789, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,014 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,014 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434009) roll requested 2024-11-24T02:47:14,014 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434014 2024-11-24T02:47:14,019 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,019 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,019 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,019 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,020 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,020 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434009 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434014 2024-11-24T02:47:14,020 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,020 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,020 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,021 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=790, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,021 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,021 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434014) roll requested 2024-11-24T02:47:14,021 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434021 2024-11-24T02:47:14,026 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,026 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,026 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,026 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,026 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,026 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434014 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434021 2024-11-24T02:47:14,027 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,027 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,027 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,028 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=791, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,028 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,028 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434021) roll requested 2024-11-24T02:47:14,028 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434028 2024-11-24T02:47:14,033 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,033 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,033 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,034 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,034 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,034 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434021 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434028 2024-11-24T02:47:14,034 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,034 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,034 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,035 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=792, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,035 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,035 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434028) roll requested 2024-11-24T02:47:14,035 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434035 2024-11-24T02:47:14,041 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,041 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,041 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,041 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,041 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,041 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434028 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434035 2024-11-24T02:47:14,042 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,042 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,042 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,043 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=793, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,043 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,043 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434035) roll requested 2024-11-24T02:47:14,043 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434043 2024-11-24T02:47:14,050 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,050 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,050 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,050 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,050 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,050 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434035 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434043 2024-11-24T02:47:14,051 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,051 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,051 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,051 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=794, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,052 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,052 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434043) roll requested 2024-11-24T02:47:14,052 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434052 2024-11-24T02:47:14,059 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,059 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,059 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,059 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,059 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,059 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434043 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434052 2024-11-24T02:47:14,060 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,060 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,060 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,060 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=795, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,060 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,060 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434052) roll requested 2024-11-24T02:47:14,060 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434060 2024-11-24T02:47:14,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,066 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434052 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434060 2024-11-24T02:47:14,067 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,067 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,067 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,067 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=796, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,067 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,067 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434060) roll requested 2024-11-24T02:47:14,068 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434068 2024-11-24T02:47:14,073 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,073 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,073 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,073 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,073 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,073 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434060 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434068 2024-11-24T02:47:14,074 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,074 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=797, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434068) roll requested 2024-11-24T02:47:14,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434075 2024-11-24T02:47:14,082 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,082 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,082 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,082 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,082 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,083 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434068 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434075 2024-11-24T02:47:14,083 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,084 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,084 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,084 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=798, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,084 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,084 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434075) roll requested 2024-11-24T02:47:14,084 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434084 2024-11-24T02:47:14,092 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,092 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,092 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,092 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,092 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,092 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434084 2024-11-24T02:47:14,093 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,093 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,093 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,093 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=799, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,094 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,094 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434084) roll requested 2024-11-24T02:47:14,094 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434094 2024-11-24T02:47:14,100 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,100 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,100 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,100 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,100 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,100 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434084 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434094 2024-11-24T02:47:14,101 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,101 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=800, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434094) roll requested 2024-11-24T02:47:14,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434101 2024-11-24T02:47:14,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,108 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,108 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434094 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434101 2024-11-24T02:47:14,109 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,109 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,109 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,109 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=801, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,109 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,110 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434101) roll requested 2024-11-24T02:47:14,110 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434110 2024-11-24T02:47:14,116 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,116 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,116 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,117 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,117 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,117 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434110 2024-11-24T02:47:14,118 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,118 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,118 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,118 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=802, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,118 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,118 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434110) roll requested 2024-11-24T02:47:14,118 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434118 2024-11-24T02:47:14,125 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,125 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,125 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,125 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,125 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,126 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434110 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434118 2024-11-24T02:47:14,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,127 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,127 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=803, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,127 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,127 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434118) roll requested 2024-11-24T02:47:14,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434128 2024-11-24T02:47:14,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,135 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434118 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434128 2024-11-24T02:47:14,136 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,136 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,136 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,136 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=804, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434128) roll requested 2024-11-24T02:47:14,137 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434137 2024-11-24T02:47:14,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,144 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,144 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434137 2024-11-24T02:47:14,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=805, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,146 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,146 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434137) roll requested 2024-11-24T02:47:14,146 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434146 2024-11-24T02:47:14,151 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,151 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434137 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434146 2024-11-24T02:47:14,152 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,152 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,152 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=806, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,153 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,153 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434146) roll requested 2024-11-24T02:47:14,153 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434153 2024-11-24T02:47:14,158 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,158 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,158 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,158 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,158 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,158 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434146 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434153 2024-11-24T02:47:14,159 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,159 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,159 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,159 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=807, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,160 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,160 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434153) roll requested 2024-11-24T02:47:14,160 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434160 2024-11-24T02:47:14,164 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,164 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,164 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,164 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434153 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434160 2024-11-24T02:47:14,165 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,165 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=808, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,165 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,244 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434160) roll requested 2024-11-24T02:47:14,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434245 2024-11-24T02:47:14,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,255 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,255 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,255 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,255 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434160 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434245 2024-11-24T02:47:14,257 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,257 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,257 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,257 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=809, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,257 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,258 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434245) roll requested 2024-11-24T02:47:14,258 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434258 2024-11-24T02:47:14,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434258 2024-11-24T02:47:14,267 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,267 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,268 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=810, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,268 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,268 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434258) roll requested 2024-11-24T02:47:14,268 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434268 2024-11-24T02:47:14,274 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,274 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,275 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,275 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,275 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,275 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434258 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434268 2024-11-24T02:47:14,276 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,276 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,276 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,276 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=811, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,276 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,277 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434268) roll requested 2024-11-24T02:47:14,277 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434277 2024-11-24T02:47:14,282 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,282 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,283 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,283 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,283 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,283 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434268 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434277 2024-11-24T02:47:14,284 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,284 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,284 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,284 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=812, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,284 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,284 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434277) roll requested 2024-11-24T02:47:14,284 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434284 2024-11-24T02:47:14,290 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,290 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,290 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,290 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,291 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,291 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434277 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434284 2024-11-24T02:47:14,292 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,292 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,292 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,292 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=813, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,292 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,292 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434284) roll requested 2024-11-24T02:47:14,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434293 2024-11-24T02:47:14,300 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,300 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,300 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,300 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,300 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,300 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434284 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434293 2024-11-24T02:47:14,301 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,301 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,301 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,301 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=814, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,301 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,301 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434293) roll requested 2024-11-24T02:47:14,301 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434301 2024-11-24T02:47:14,307 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,307 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,307 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434301 2024-11-24T02:47:14,310 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,310 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,311 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,311 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=815, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,311 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,311 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434301) roll requested 2024-11-24T02:47:14,311 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434311 2024-11-24T02:47:14,317 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,317 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,318 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,318 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,318 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,318 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434301 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434311 2024-11-24T02:47:14,318 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,318 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,318 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,319 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=816, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,319 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,319 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434311) roll requested 2024-11-24T02:47:14,319 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434319 2024-11-24T02:47:14,324 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,324 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,324 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,324 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,324 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,325 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434311 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434319 2024-11-24T02:47:14,325 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,325 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,325 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,325 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=817, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,325 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,325 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434319) roll requested 2024-11-24T02:47:14,326 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434325 2024-11-24T02:47:14,332 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,332 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,332 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,332 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,332 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,332 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434319 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434325 2024-11-24T02:47:14,333 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,333 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,333 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,333 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=818, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,334 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434325) roll requested 2024-11-24T02:47:14,334 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434334 2024-11-24T02:47:14,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434325 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434334 2024-11-24T02:47:14,346 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,346 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,346 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,347 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=819, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,347 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,347 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434334) roll requested 2024-11-24T02:47:14,347 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434347 2024-11-24T02:47:14,353 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,353 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,353 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,353 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,353 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,353 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434334 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434347 2024-11-24T02:47:14,354 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,354 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,354 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,354 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=820, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,354 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,354 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434347) roll requested 2024-11-24T02:47:14,355 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434354 2024-11-24T02:47:14,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,360 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434347 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434354 2024-11-24T02:47:14,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,361 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,361 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,361 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=821, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,361 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,361 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434354) roll requested 2024-11-24T02:47:14,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434361 2024-11-24T02:47:14,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434354 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434361 2024-11-24T02:47:14,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=822, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434361) roll requested 2024-11-24T02:47:14,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434368 2024-11-24T02:47:14,374 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,374 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,375 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,375 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,375 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434361 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434368 2024-11-24T02:47:14,375 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,375 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,375 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,376 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=823, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,376 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,376 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434368) roll requested 2024-11-24T02:47:14,376 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434376 2024-11-24T02:47:14,383 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,383 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,383 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,383 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,383 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,383 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434376 2024-11-24T02:47:14,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=824, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,384 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434376) roll requested 2024-11-24T02:47:14,384 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434384 2024-11-24T02:47:14,390 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,390 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,390 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,390 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,390 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,390 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434376 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434384 2024-11-24T02:47:14,391 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,391 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,391 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,392 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=825, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,392 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,392 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434384) roll requested 2024-11-24T02:47:14,392 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434392 2024-11-24T02:47:14,398 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,399 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,399 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,399 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,399 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,399 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434384 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434392 2024-11-24T02:47:14,400 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,400 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,400 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,400 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=826, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,400 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,400 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434392) roll requested 2024-11-24T02:47:14,400 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434400 2024-11-24T02:47:14,406 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,407 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,407 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,407 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,407 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,407 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434392 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434400 2024-11-24T02:47:14,408 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,408 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,408 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,408 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=827, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,408 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,408 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434400) roll requested 2024-11-24T02:47:14,408 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434408 2024-11-24T02:47:14,414 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,414 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,414 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,414 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,414 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,414 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434400 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434408 2024-11-24T02:47:14,415 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,415 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,415 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=828, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,416 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434408) roll requested 2024-11-24T02:47:14,416 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434416 2024-11-24T02:47:14,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434408 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434416 2024-11-24T02:47:14,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,424 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=829, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,424 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,424 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434416) roll requested 2024-11-24T02:47:14,424 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434424 2024-11-24T02:47:14,431 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,431 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,432 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434416 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434424 2024-11-24T02:47:14,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=830, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,433 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,433 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434424) roll requested 2024-11-24T02:47:14,433 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434433 2024-11-24T02:47:14,441 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,442 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434424 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434433 2024-11-24T02:47:14,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,444 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=831, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434433) roll requested 2024-11-24T02:47:14,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434444 2024-11-24T02:47:14,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434433 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434444 2024-11-24T02:47:14,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=832, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434444) roll requested 2024-11-24T02:47:14,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434452 2024-11-24T02:47:14,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434452 2024-11-24T02:47:14,458 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,458 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,458 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=833, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,459 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,459 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434452) roll requested 2024-11-24T02:47:14,459 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434459 2024-11-24T02:47:14,464 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,464 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,464 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,464 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,465 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434459 2024-11-24T02:47:14,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,466 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=834, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,466 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,466 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434459) roll requested 2024-11-24T02:47:14,466 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434466 2024-11-24T02:47:14,473 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,473 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,474 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,474 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,474 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,474 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434459 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434466 2024-11-24T02:47:14,475 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,475 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,475 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,475 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=835, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,475 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,475 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434466) roll requested 2024-11-24T02:47:14,475 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434475 2024-11-24T02:47:14,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,482 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,482 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434466 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434475 2024-11-24T02:47:14,482 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,482 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,482 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,483 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=836, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,483 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,483 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434475) roll requested 2024-11-24T02:47:14,483 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434483 2024-11-24T02:47:14,488 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,488 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,488 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434475 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434483 2024-11-24T02:47:14,489 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,489 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=837, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434483) roll requested 2024-11-24T02:47:14,490 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434490 2024-11-24T02:47:14,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434483 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434490 2024-11-24T02:47:14,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=838, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434490) roll requested 2024-11-24T02:47:14,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434497 2024-11-24T02:47:14,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434490 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434497 2024-11-24T02:47:14,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,505 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,505 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=839, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,505 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,505 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434497) roll requested 2024-11-24T02:47:14,505 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434505 2024-11-24T02:47:14,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434505 2024-11-24T02:47:14,515 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,515 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,515 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,515 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=840, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434505) roll requested 2024-11-24T02:47:14,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434516 2024-11-24T02:47:14,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434505 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434516 2024-11-24T02:47:14,522 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=841, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434516) roll requested 2024-11-24T02:47:14,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434523 2024-11-24T02:47:14,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,528 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434523 2024-11-24T02:47:14,528 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,528 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,528 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,528 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=842, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,528 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,528 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434523) roll requested 2024-11-24T02:47:14,528 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434528 2024-11-24T02:47:14,533 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,533 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,533 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,533 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,533 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,534 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434528 2024-11-24T02:47:14,534 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,534 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,534 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,535 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=843, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,535 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,535 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434528) roll requested 2024-11-24T02:47:14,535 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434535 2024-11-24T02:47:14,540 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,540 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,541 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,541 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,541 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,541 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434528 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434535 2024-11-24T02:47:14,542 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,542 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,542 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,543 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=844, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,543 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,543 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434535) roll requested 2024-11-24T02:47:14,543 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434543 2024-11-24T02:47:14,549 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,549 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,549 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,550 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,550 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,550 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434535 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434543 2024-11-24T02:47:14,551 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,551 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,551 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,551 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=845, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,551 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,551 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434543) roll requested 2024-11-24T02:47:14,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434551 2024-11-24T02:47:14,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,559 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434543 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434551 2024-11-24T02:47:14,560 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,560 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,560 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,560 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=846, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,561 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,561 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434551) roll requested 2024-11-24T02:47:14,561 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434561 2024-11-24T02:47:14,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,570 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434551 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434561 2024-11-24T02:47:14,571 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,571 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,571 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,571 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=847, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,571 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,571 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434561) roll requested 2024-11-24T02:47:14,571 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434571 2024-11-24T02:47:14,578 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,578 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,579 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,579 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,579 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,579 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434561 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434571 2024-11-24T02:47:14,580 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,580 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,580 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=848, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,580 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,581 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,581 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434571) roll requested 2024-11-24T02:47:14,581 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434581 2024-11-24T02:47:14,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,588 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,588 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,588 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,588 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434571 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434581 2024-11-24T02:47:14,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,589 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,589 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=849, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434581) roll requested 2024-11-24T02:47:14,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434589 2024-11-24T02:47:14,594 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,594 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,594 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,594 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,594 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434581 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434589 2024-11-24T02:47:14,595 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,596 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=850, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,596 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,596 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434589) roll requested 2024-11-24T02:47:14,596 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434596 2024-11-24T02:47:14,601 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,602 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,602 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,602 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,602 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,602 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434596 2024-11-24T02:47:14,603 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,603 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,603 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,604 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=851, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,604 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,604 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434596) roll requested 2024-11-24T02:47:14,604 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434604 2024-11-24T02:47:14,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,612 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434596 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434604 2024-11-24T02:47:14,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,613 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=852, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434604) roll requested 2024-11-24T02:47:14,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434613 2024-11-24T02:47:14,618 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,618 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,619 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,619 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,619 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,619 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434604 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434613 2024-11-24T02:47:14,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,620 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=853, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,620 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,620 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434613) roll requested 2024-11-24T02:47:14,620 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434620 2024-11-24T02:47:14,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,627 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,627 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,627 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434620 2024-11-24T02:47:14,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,627 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,627 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=854, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434620) roll requested 2024-11-24T02:47:14,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434628 2024-11-24T02:47:14,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,635 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,635 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434620 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434628 2024-11-24T02:47:14,636 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,636 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,636 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,636 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=855, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,636 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,636 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434628) roll requested 2024-11-24T02:47:14,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434637 2024-11-24T02:47:14,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434637 2024-11-24T02:47:14,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,645 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=856, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434637) roll requested 2024-11-24T02:47:14,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434645 2024-11-24T02:47:14,653 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,653 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,653 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,653 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,654 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,654 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434645 2024-11-24T02:47:14,654 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,655 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,655 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,655 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=857, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,655 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,655 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434645) roll requested 2024-11-24T02:47:14,655 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434655 2024-11-24T02:47:14,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434655 2024-11-24T02:47:14,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=858, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434655) roll requested 2024-11-24T02:47:14,665 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434665 2024-11-24T02:47:14,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434655 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434665 2024-11-24T02:47:14,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,676 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=859, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434665) roll requested 2024-11-24T02:47:14,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434676 2024-11-24T02:47:14,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434665 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434676 2024-11-24T02:47:14,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=860, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434676) roll requested 2024-11-24T02:47:14,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434681 2024-11-24T02:47:14,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434681 2024-11-24T02:47:14,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=861, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,687 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,687 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434681) roll requested 2024-11-24T02:47:14,687 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434687 2024-11-24T02:47:14,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,693 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434687 2024-11-24T02:47:14,693 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,693 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=862, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434687) roll requested 2024-11-24T02:47:14,694 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434693 2024-11-24T02:47:14,699 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,699 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,700 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434687 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434693 2024-11-24T02:47:14,700 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,700 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,701 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,701 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=863, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,701 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,701 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434693) roll requested 2024-11-24T02:47:14,701 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434701 2024-11-24T02:47:14,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434701 2024-11-24T02:47:14,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=864, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,709 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,709 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434701) roll requested 2024-11-24T02:47:14,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434709 2024-11-24T02:47:14,715 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,715 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,715 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,715 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,715 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,715 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434701 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434709 2024-11-24T02:47:14,716 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,716 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,717 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,717 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=865, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,717 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,717 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434709) roll requested 2024-11-24T02:47:14,717 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434717 2024-11-24T02:47:14,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434717 2024-11-24T02:47:14,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=866, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,726 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434717) roll requested 2024-11-24T02:47:14,726 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434726 2024-11-24T02:47:14,732 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,732 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,733 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,733 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,733 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,733 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434717 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434726 2024-11-24T02:47:14,733 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,734 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,734 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,734 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=867, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,734 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,734 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434726) roll requested 2024-11-24T02:47:14,734 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434734 2024-11-24T02:47:14,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434726 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434734 2024-11-24T02:47:14,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,743 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=868, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,743 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,743 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434734) roll requested 2024-11-24T02:47:14,743 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434743 2024-11-24T02:47:14,750 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,750 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,750 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434734 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434743 2024-11-24T02:47:14,751 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,751 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,751 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=869, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434743) roll requested 2024-11-24T02:47:14,752 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434751 2024-11-24T02:47:14,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434743 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434751 2024-11-24T02:47:14,760 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,760 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,760 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,760 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=870, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,760 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,760 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434751) roll requested 2024-11-24T02:47:14,761 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434761 2024-11-24T02:47:14,765 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,766 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,766 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,766 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,766 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,766 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434761 2024-11-24T02:47:14,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,766 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=871, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434761) roll requested 2024-11-24T02:47:14,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434767 2024-11-24T02:47:14,771 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,771 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,771 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,771 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,771 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,771 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434761 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434767 2024-11-24T02:47:14,771 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,771 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,772 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,772 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=872, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,772 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,772 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434767) roll requested 2024-11-24T02:47:14,772 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434772 2024-11-24T02:47:14,777 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,777 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,777 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,777 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,777 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,777 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434772 2024-11-24T02:47:14,778 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,778 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,778 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,778 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=873, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,778 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,778 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434772) roll requested 2024-11-24T02:47:14,778 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434778 2024-11-24T02:47:14,783 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,783 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,783 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,783 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,784 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,784 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434772 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434778 2024-11-24T02:47:14,784 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,784 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,784 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,784 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=874, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,785 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,785 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434778) roll requested 2024-11-24T02:47:14,785 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434785 2024-11-24T02:47:14,790 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,790 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,790 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,791 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,791 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,791 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434778 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434785 2024-11-24T02:47:14,792 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,792 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,792 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,792 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=875, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,792 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,792 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434785) roll requested 2024-11-24T02:47:14,792 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434792 2024-11-24T02:47:14,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434785 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434792 2024-11-24T02:47:14,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=876, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434792) roll requested 2024-11-24T02:47:14,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434800 2024-11-24T02:47:14,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,807 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434792 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434800 2024-11-24T02:47:14,808 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=877, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434800) roll requested 2024-11-24T02:47:14,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434808 2024-11-24T02:47:14,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,816 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,816 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,816 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434808 2024-11-24T02:47:14,817 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,817 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,817 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=878, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434808) roll requested 2024-11-24T02:47:14,818 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434818 2024-11-24T02:47:14,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434818 2024-11-24T02:47:14,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=879, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434818) roll requested 2024-11-24T02:47:14,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434825 2024-11-24T02:47:14,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434818 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434825 2024-11-24T02:47:14,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=880, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,832 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434825) roll requested 2024-11-24T02:47:14,832 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434832 2024-11-24T02:47:14,838 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,838 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,838 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,839 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,839 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,839 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434832 2024-11-24T02:47:14,840 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,840 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,840 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,840 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=881, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,840 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,840 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434832) roll requested 2024-11-24T02:47:14,841 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434840 2024-11-24T02:47:14,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,848 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,848 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434832 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434840 2024-11-24T02:47:14,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=882, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434840) roll requested 2024-11-24T02:47:14,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434850 2024-11-24T02:47:14,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,858 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,858 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,859 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434840 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434850 2024-11-24T02:47:14,859 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,859 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,860 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,860 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=883, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,860 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,860 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434850) roll requested 2024-11-24T02:47:14,860 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434860 2024-11-24T02:47:14,868 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,868 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,868 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,868 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434860 2024-11-24T02:47:14,869 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,869 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,869 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,869 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=884, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434860) roll requested 2024-11-24T02:47:14,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434870 2024-11-24T02:47:14,878 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,878 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,878 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,878 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,878 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,878 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434860 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434870 2024-11-24T02:47:14,880 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,880 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,880 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,880 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=885, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,880 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,880 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434870) roll requested 2024-11-24T02:47:14,880 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434880 2024-11-24T02:47:14,886 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,886 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,886 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,886 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,887 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,887 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434880 2024-11-24T02:47:14,888 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,888 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,888 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,888 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=886, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,888 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,888 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434880) roll requested 2024-11-24T02:47:14,888 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434888 2024-11-24T02:47:14,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434880 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434888 2024-11-24T02:47:14,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=887, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434888) roll requested 2024-11-24T02:47:14,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434897 2024-11-24T02:47:14,904 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,904 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,904 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,905 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,905 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,905 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434888 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434897 2024-11-24T02:47:14,906 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,906 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,906 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,906 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=888, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,906 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,906 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434897) roll requested 2024-11-24T02:47:14,906 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434906 2024-11-24T02:47:14,914 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,914 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,914 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,914 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,915 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,915 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434906 2024-11-24T02:47:14,916 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,916 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,916 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,916 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=889, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,916 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434906) roll requested 2024-11-24T02:47:14,917 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434917 2024-11-24T02:47:14,922 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,922 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,922 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,922 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,922 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,922 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434906 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434917 2024-11-24T02:47:14,923 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,923 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,923 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,924 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=890, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,924 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,924 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434917) roll requested 2024-11-24T02:47:14,924 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434924 2024-11-24T02:47:14,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434917 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434924 2024-11-24T02:47:14,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=891, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434924) roll requested 2024-11-24T02:47:14,932 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434931 2024-11-24T02:47:14,938 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,938 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,938 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,938 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,938 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,938 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434924 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434931 2024-11-24T02:47:14,939 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,939 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,939 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,940 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=892, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,940 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,940 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434931) roll requested 2024-11-24T02:47:14,940 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434940 2024-11-24T02:47:14,947 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,947 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434940 2024-11-24T02:47:14,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=893, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434940) roll requested 2024-11-24T02:47:14,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434950 2024-11-24T02:47:14,958 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,958 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,958 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,958 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,958 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,958 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434940 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434950 2024-11-24T02:47:14,960 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,960 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,960 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,960 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=894, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,960 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,960 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434950) roll requested 2024-11-24T02:47:14,960 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434960 2024-11-24T02:47:14,968 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,968 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,968 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,968 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,969 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,969 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434950 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434960 2024-11-24T02:47:14,970 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,970 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,970 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,970 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=895, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,970 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,970 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434960) roll requested 2024-11-24T02:47:14,970 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434970 2024-11-24T02:47:14,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434960 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434970 2024-11-24T02:47:14,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=896, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434970) roll requested 2024-11-24T02:47:14,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434979 2024-11-24T02:47:14,985 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,985 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,985 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,985 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,985 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,985 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434970 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434979 2024-11-24T02:47:14,987 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,987 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,987 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,987 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=897, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,987 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,987 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434979) roll requested 2024-11-24T02:47:14,987 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434987 2024-11-24T02:47:14,995 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,995 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,995 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,995 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,995 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:14,995 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434987 2024-11-24T02:47:14,996 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:14,996 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,997 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:14,997 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=898, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:14,997 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:14,997 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434987) roll requested 2024-11-24T02:47:14,997 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416434997 2024-11-24T02:47:15,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434987 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434997 2024-11-24T02:47:15,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,006 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=899, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,006 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,006 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416434997) roll requested 2024-11-24T02:47:15,006 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435006 2024-11-24T02:47:15,013 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,013 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,013 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,013 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,013 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,013 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416434997 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435006 2024-11-24T02:47:15,014 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,014 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,015 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,015 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=900, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,015 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,015 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435006) roll requested 2024-11-24T02:47:15,015 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435015 2024-11-24T02:47:15,022 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,022 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,022 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,022 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,022 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,022 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435006 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435015 2024-11-24T02:47:15,023 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,023 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,023 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,023 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=901, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,023 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,024 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435015) roll requested 2024-11-24T02:47:15,024 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435024 2024-11-24T02:47:15,031 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,031 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,031 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,031 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,031 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,031 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435015 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435024 2024-11-24T02:47:15,032 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,032 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,032 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,032 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=902, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,032 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,032 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435024) roll requested 2024-11-24T02:47:15,032 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435032 2024-11-24T02:47:15,038 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,038 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,038 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,038 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,038 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,038 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435024 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435032 2024-11-24T02:47:15,039 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,039 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,039 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,039 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=903, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435032) roll requested 2024-11-24T02:47:15,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435040 2024-11-24T02:47:15,046 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,046 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,046 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,046 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,046 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,047 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435032 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435040 2024-11-24T02:47:15,047 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,047 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,047 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,048 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=904, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,048 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,048 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435040) roll requested 2024-11-24T02:47:15,048 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435048 2024-11-24T02:47:15,054 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,054 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435048 2024-11-24T02:47:15,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=905, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,057 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435048) roll requested 2024-11-24T02:47:15,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435057 2024-11-24T02:47:15,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,064 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,064 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,064 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,064 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,064 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435048 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435057 2024-11-24T02:47:15,065 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,065 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,065 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,065 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=906, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,065 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,065 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435057) roll requested 2024-11-24T02:47:15,066 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435066 2024-11-24T02:47:15,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435066 2024-11-24T02:47:15,076 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,076 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,076 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,076 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=907, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,076 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,076 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435066) roll requested 2024-11-24T02:47:15,076 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435076 2024-11-24T02:47:15,085 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,085 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,085 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,085 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,085 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,085 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435066 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435076 2024-11-24T02:47:15,086 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,086 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=908, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,087 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435076) roll requested 2024-11-24T02:47:15,087 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435087 2024-11-24T02:47:15,093 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,093 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,093 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,093 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,093 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,093 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435076 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435087 2024-11-24T02:47:15,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,097 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=909, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,097 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,097 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435087) roll requested 2024-11-24T02:47:15,097 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435097 2024-11-24T02:47:15,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,104 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435087 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435097 2024-11-24T02:47:15,105 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,105 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,105 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,105 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=910, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,106 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,106 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435097) roll requested 2024-11-24T02:47:15,106 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435106 2024-11-24T02:47:15,112 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,113 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,113 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,113 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,113 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,113 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435097 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435106 2024-11-24T02:47:15,114 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,114 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,114 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,114 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=911, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,114 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,114 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435106) roll requested 2024-11-24T02:47:15,115 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435114 2024-11-24T02:47:15,122 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,123 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,123 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,123 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435106 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435114 2024-11-24T02:47:15,124 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,124 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,124 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,124 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=912, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,124 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,124 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435114) roll requested 2024-11-24T02:47:15,125 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435125 2024-11-24T02:47:15,133 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,133 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,133 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,133 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,133 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,133 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435114 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435125 2024-11-24T02:47:15,134 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,134 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=913, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,135 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,135 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435125) roll requested 2024-11-24T02:47:15,135 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435135 2024-11-24T02:47:15,143 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,143 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,144 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,144 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,144 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435125 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435135 2024-11-24T02:47:15,145 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,145 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,145 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,145 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=914, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435135) roll requested 2024-11-24T02:47:15,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435145 2024-11-24T02:47:15,150 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,150 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,150 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,150 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,150 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,150 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435135 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435145 2024-11-24T02:47:15,151 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,151 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,151 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,151 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=915, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,151 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,151 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435145) roll requested 2024-11-24T02:47:15,151 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435151 2024-11-24T02:47:15,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435151 2024-11-24T02:47:15,157 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,157 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,157 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,157 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=916, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,157 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,157 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435151) roll requested 2024-11-24T02:47:15,158 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435157 2024-11-24T02:47:15,163 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,163 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,163 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,163 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,164 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,164 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435151 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435157 2024-11-24T02:47:15,168 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,168 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,169 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,169 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=917, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,169 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,169 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435157) roll requested 2024-11-24T02:47:15,169 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435169 2024-11-24T02:47:15,175 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,175 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,176 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,176 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,176 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,176 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435157 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435169 2024-11-24T02:47:15,181 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,181 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,181 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,181 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=918, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,181 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,181 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435169) roll requested 2024-11-24T02:47:15,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435181 2024-11-24T02:47:15,188 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,188 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,188 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,188 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,188 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,188 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435169 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435181 2024-11-24T02:47:15,191 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,191 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,191 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,192 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=919, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,192 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,192 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435181) roll requested 2024-11-24T02:47:15,192 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435192 2024-11-24T02:47:15,196 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,196 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,197 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,197 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,197 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,197 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435181 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435192 2024-11-24T02:47:15,197 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,197 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,197 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,198 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=920, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,198 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,198 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435192) roll requested 2024-11-24T02:47:15,198 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435198 2024-11-24T02:47:15,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,202 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435192 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435198 2024-11-24T02:47:15,203 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,203 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,203 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,203 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=921, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,203 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,203 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435198) roll requested 2024-11-24T02:47:15,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435203 2024-11-24T02:47:15,210 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,210 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,210 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,210 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,210 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,210 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435198 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435203 2024-11-24T02:47:15,211 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,211 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,211 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,211 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=922, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,211 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,211 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435203) roll requested 2024-11-24T02:47:15,211 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435211 2024-11-24T02:47:15,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,216 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435203 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435211 2024-11-24T02:47:15,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=923, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435211) roll requested 2024-11-24T02:47:15,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435217 2024-11-24T02:47:15,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,222 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,222 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,222 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435211 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435217 2024-11-24T02:47:15,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,222 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,222 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=924, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435217) roll requested 2024-11-24T02:47:15,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435223 2024-11-24T02:47:15,227 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,228 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,228 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,228 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,228 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435223 2024-11-24T02:47:15,228 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,229 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,229 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,229 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=925, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,229 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,229 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435223) roll requested 2024-11-24T02:47:15,229 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435229 2024-11-24T02:47:15,235 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,235 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,235 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,235 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,235 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,235 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435229 2024-11-24T02:47:15,236 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,236 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,236 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,237 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=926, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,237 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,237 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435229) roll requested 2024-11-24T02:47:15,237 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435237 2024-11-24T02:47:15,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,245 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,245 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,245 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435229 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435237 2024-11-24T02:47:15,246 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,246 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,246 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,246 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=927, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,247 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,247 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435237) roll requested 2024-11-24T02:47:15,247 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435247 2024-11-24T02:47:15,254 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,254 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,254 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,254 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,254 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,255 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435237 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435247 2024-11-24T02:47:15,256 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,256 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,256 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,256 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=928, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,256 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,256 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435247) roll requested 2024-11-24T02:47:15,256 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435256 2024-11-24T02:47:15,263 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,263 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,263 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,263 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,263 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,263 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435247 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435256 2024-11-24T02:47:15,264 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,264 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,264 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,265 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=929, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,265 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,265 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435256) roll requested 2024-11-24T02:47:15,265 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435265 2024-11-24T02:47:15,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435256 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435265 2024-11-24T02:47:15,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=930, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,273 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435265) roll requested 2024-11-24T02:47:15,273 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435273 2024-11-24T02:47:15,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,280 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,280 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,280 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,280 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435265 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435273 2024-11-24T02:47:15,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=931, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435273) roll requested 2024-11-24T02:47:15,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435281 2024-11-24T02:47:15,289 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,289 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,289 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,289 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,289 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,289 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435273 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435281 2024-11-24T02:47:15,290 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,290 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,290 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,291 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=932, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,291 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,291 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435281) roll requested 2024-11-24T02:47:15,291 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435291 2024-11-24T02:47:15,299 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,299 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,299 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,299 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,299 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,299 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435291 2024-11-24T02:47:15,300 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,300 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,300 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,300 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=933, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,300 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,300 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435291) roll requested 2024-11-24T02:47:15,300 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435300 2024-11-24T02:47:15,305 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,305 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,305 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,306 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,306 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,306 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435291 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435300 2024-11-24T02:47:15,306 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,307 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,307 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,307 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=934, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,307 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,307 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435300) roll requested 2024-11-24T02:47:15,307 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435307 2024-11-24T02:47:15,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435300 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435307 2024-11-24T02:47:15,314 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,314 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,314 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=935, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435307) roll requested 2024-11-24T02:47:15,315 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435315 2024-11-24T02:47:15,321 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,321 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435307 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435315 2024-11-24T02:47:15,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=936, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,323 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,323 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435315) roll requested 2024-11-24T02:47:15,323 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435323 2024-11-24T02:47:15,330 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,331 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435315 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435323 2024-11-24T02:47:15,332 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,332 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,332 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,332 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=937, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,333 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,333 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435323) roll requested 2024-11-24T02:47:15,333 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435333 2024-11-24T02:47:15,339 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,340 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,340 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,340 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,340 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,340 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435323 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435333 2024-11-24T02:47:15,341 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,341 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,341 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,341 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=938, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,341 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,341 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435333) roll requested 2024-11-24T02:47:15,341 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435341 2024-11-24T02:47:15,348 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,349 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,349 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435333 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435341 2024-11-24T02:47:15,349 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,349 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,349 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,350 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=939, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,350 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,350 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435341) roll requested 2024-11-24T02:47:15,350 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435350 2024-11-24T02:47:15,357 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,357 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,357 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,357 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,357 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,358 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435341 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435350 2024-11-24T02:47:15,359 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,359 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,359 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,359 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=940, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,359 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435350) roll requested 2024-11-24T02:47:15,360 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435360 2024-11-24T02:47:15,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,367 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,367 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,367 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,367 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435350 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435360 2024-11-24T02:47:15,368 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,368 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,368 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,368 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=941, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,368 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,368 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435360) roll requested 2024-11-24T02:47:15,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435368 2024-11-24T02:47:15,376 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,376 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,376 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,376 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,376 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,377 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435368 2024-11-24T02:47:15,377 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,377 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,378 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,378 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=942, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,378 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,378 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435368) roll requested 2024-11-24T02:47:15,378 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435378 2024-11-24T02:47:15,385 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,385 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,385 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,386 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,386 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,386 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435378 2024-11-24T02:47:15,386 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,386 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,386 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,387 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=943, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,387 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,387 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435378) roll requested 2024-11-24T02:47:15,387 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435387 2024-11-24T02:47:15,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,392 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435378 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435387 2024-11-24T02:47:15,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,396 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,396 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=944, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435387) roll requested 2024-11-24T02:47:15,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435397 2024-11-24T02:47:15,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435387 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435397 2024-11-24T02:47:15,403 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,403 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,403 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,403 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=945, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,403 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,403 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435397) roll requested 2024-11-24T02:47:15,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435403 2024-11-24T02:47:15,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,408 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,408 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,408 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,408 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,408 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435403 2024-11-24T02:47:15,411 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,411 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,411 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,411 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=946, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,412 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,412 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435403) roll requested 2024-11-24T02:47:15,412 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435412 2024-11-24T02:47:15,416 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,416 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,416 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,416 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,416 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,416 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435403 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435412 2024-11-24T02:47:15,417 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,417 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,417 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,417 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=947, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435412) roll requested 2024-11-24T02:47:15,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435417 2024-11-24T02:47:15,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435412 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435417 2024-11-24T02:47:15,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=948, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435417) roll requested 2024-11-24T02:47:15,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435423 2024-11-24T02:47:15,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435423 2024-11-24T02:47:15,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=949, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435423) roll requested 2024-11-24T02:47:15,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435428 2024-11-24T02:47:15,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,432 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,432 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,432 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,432 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435428 2024-11-24T02:47:15,433 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,433 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,433 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,433 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=950, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435428) roll requested 2024-11-24T02:47:15,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435434 2024-11-24T02:47:15,438 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,438 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,438 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,438 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,438 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,439 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435434 2024-11-24T02:47:15,440 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,440 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,440 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,440 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=951, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,440 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,440 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435434) roll requested 2024-11-24T02:47:15,441 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435441 2024-11-24T02:47:15,445 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,445 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,445 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,445 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,445 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435441 2024-11-24T02:47:15,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=952, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,446 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,446 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435441) roll requested 2024-11-24T02:47:15,446 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435446 2024-11-24T02:47:15,451 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,451 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,451 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,451 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,451 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,451 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435441 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435446 2024-11-24T02:47:15,451 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=953, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,452 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,452 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435446) roll requested 2024-11-24T02:47:15,452 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435452 2024-11-24T02:47:15,456 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,456 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435446 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435452 2024-11-24T02:47:15,457 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,457 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,457 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,458 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=954, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,458 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,458 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435452) roll requested 2024-11-24T02:47:15,458 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435458 2024-11-24T02:47:15,463 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,463 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,463 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,463 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,463 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,463 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435452 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435458 2024-11-24T02:47:15,464 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,464 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,464 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,464 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=955, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,464 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,464 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435458) roll requested 2024-11-24T02:47:15,465 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435465 2024-11-24T02:47:15,469 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,469 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,469 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,469 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,469 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,469 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435458 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435465 2024-11-24T02:47:15,470 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,470 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,470 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,470 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=956, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,470 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,470 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435465) roll requested 2024-11-24T02:47:15,470 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435470 2024-11-24T02:47:15,475 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,475 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,475 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,475 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,475 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,475 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435465 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435470 2024-11-24T02:47:15,476 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,476 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,476 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=957, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,476 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,476 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,476 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435470) roll requested 2024-11-24T02:47:15,477 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435476 2024-11-24T02:47:15,481 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,481 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,481 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,481 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,481 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,481 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435470 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435476 2024-11-24T02:47:15,484 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,484 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,484 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,484 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=958, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,484 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,484 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435476) roll requested 2024-11-24T02:47:15,485 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435485 2024-11-24T02:47:15,489 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,489 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,489 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,489 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,489 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,489 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435476 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435485 2024-11-24T02:47:15,490 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,490 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,490 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,490 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=959, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,490 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,490 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435485) roll requested 2024-11-24T02:47:15,491 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435491 2024-11-24T02:47:15,495 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,495 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,495 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,495 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,495 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,495 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435491 2024-11-24T02:47:15,496 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,496 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,496 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,496 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=960, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435491) roll requested 2024-11-24T02:47:15,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435497 2024-11-24T02:47:15,505 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,505 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,505 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,505 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,505 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,505 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435491 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435497 2024-11-24T02:47:15,506 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,506 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,506 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,506 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=961, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,506 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,506 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435497) roll requested 2024-11-24T02:47:15,507 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435507 2024-11-24T02:47:15,511 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,511 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,511 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,512 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,512 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,512 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435507 2024-11-24T02:47:15,512 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,512 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,513 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,513 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=962, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,513 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,513 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435507) roll requested 2024-11-24T02:47:15,513 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435513 2024-11-24T02:47:15,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,519 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435507 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435513 2024-11-24T02:47:15,520 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,520 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,520 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,520 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=963, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,520 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,520 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435513) roll requested 2024-11-24T02:47:15,520 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435520 2024-11-24T02:47:15,524 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,524 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,525 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,525 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,525 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435513 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435520 2024-11-24T02:47:15,525 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,525 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,526 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,526 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=964, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,526 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,526 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435520) roll requested 2024-11-24T02:47:15,526 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435526 2024-11-24T02:47:15,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,531 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435520 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435526 2024-11-24T02:47:15,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,531 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=965, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435526) roll requested 2024-11-24T02:47:15,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435532 2024-11-24T02:47:15,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,536 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,536 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435526 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435532 2024-11-24T02:47:15,537 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,537 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,537 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,537 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=966, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,537 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,537 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435532) roll requested 2024-11-24T02:47:15,538 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435538 2024-11-24T02:47:15,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,544 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,544 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435538 2024-11-24T02:47:15,545 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,545 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,545 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,546 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=967, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,546 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,546 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435538) roll requested 2024-11-24T02:47:15,546 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435546 2024-11-24T02:47:15,551 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,551 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,551 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,551 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,551 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,551 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435538 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435546 2024-11-24T02:47:15,552 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,552 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,552 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,552 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=968, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,552 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,552 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435546) roll requested 2024-11-24T02:47:15,552 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435552 2024-11-24T02:47:15,556 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435546 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435552 2024-11-24T02:47:15,557 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,557 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,557 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,558 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=969, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,558 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,558 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435552) roll requested 2024-11-24T02:47:15,558 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435558 2024-11-24T02:47:15,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,562 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,562 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,562 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435552 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435558 2024-11-24T02:47:15,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,563 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,563 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,563 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=970, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435558) roll requested 2024-11-24T02:47:15,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435564 2024-11-24T02:47:15,568 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,568 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,568 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,568 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,568 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,568 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435558 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435564 2024-11-24T02:47:15,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,569 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=971, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435564) roll requested 2024-11-24T02:47:15,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435569 2024-11-24T02:47:15,573 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,573 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,574 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,574 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,574 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,574 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435569 2024-11-24T02:47:15,574 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,575 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,575 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,575 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=972, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,575 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,575 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435569) roll requested 2024-11-24T02:47:15,575 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435575 2024-11-24T02:47:15,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435575 2024-11-24T02:47:15,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=973, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,582 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435575) roll requested 2024-11-24T02:47:15,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435583 2024-11-24T02:47:15,590 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,590 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,590 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,591 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435575 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435583 2024-11-24T02:47:15,596 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,596 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,596 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,597 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=974, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,597 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,597 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435583) roll requested 2024-11-24T02:47:15,597 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435597 2024-11-24T02:47:15,606 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,606 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435597 2024-11-24T02:47:15,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=975, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435597) roll requested 2024-11-24T02:47:15,608 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435608 2024-11-24T02:47:15,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,615 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,615 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435597 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435608 2024-11-24T02:47:15,616 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,616 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,616 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,616 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=976, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,617 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,617 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435608) roll requested 2024-11-24T02:47:15,617 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435617 2024-11-24T02:47:15,621 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,621 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,622 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,622 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,622 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,622 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435608 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435617 2024-11-24T02:47:15,623 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,623 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,623 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,623 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=977, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,623 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,623 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435617) roll requested 2024-11-24T02:47:15,623 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435623 2024-11-24T02:47:15,628 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,628 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,628 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,628 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,628 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,628 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435617 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435623 2024-11-24T02:47:15,632 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,632 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,632 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,632 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=978, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,633 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,633 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435623) roll requested 2024-11-24T02:47:15,633 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435633 2024-11-24T02:47:15,638 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,638 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,638 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,638 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,638 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,638 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435623 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435633 2024-11-24T02:47:15,639 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,639 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,639 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,639 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=979, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,639 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,639 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435633) roll requested 2024-11-24T02:47:15,639 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435639 2024-11-24T02:47:15,644 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,644 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,644 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,644 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,644 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435633 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435639 2024-11-24T02:47:15,644 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,644 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,645 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,645 DEBUG [LogRoller {}] wal.AbstractFSWAL(879): /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435633 is not closed yet, will try archiving it next time 2024-11-24T02:47:15,645 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=980, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,645 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,645 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435639) roll requested 2024-11-24T02:47:15,645 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435645 2024-11-24T02:47:15,649 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,649 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,649 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,650 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,650 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,650 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435639 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435645 2024-11-24T02:47:15,650 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,650 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,650 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,651 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=981, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,651 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,651 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435645) roll requested 2024-11-24T02:47:15,651 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435651 2024-11-24T02:47:15,655 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,655 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,655 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,655 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,655 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,655 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435645 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435651 2024-11-24T02:47:15,656 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,656 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,656 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,656 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=982, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,656 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,656 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435651) roll requested 2024-11-24T02:47:15,656 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435656 2024-11-24T02:47:15,661 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,661 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,661 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,662 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435651 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435656 2024-11-24T02:47:15,662 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,662 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,662 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,663 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=983, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,663 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,663 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435656) roll requested 2024-11-24T02:47:15,663 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435663 2024-11-24T02:47:15,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,668 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,668 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435656 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435663 2024-11-24T02:47:15,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=984, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435663) roll requested 2024-11-24T02:47:15,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435670 2024-11-24T02:47:15,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,676 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,676 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,676 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,676 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,676 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435663 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435670 2024-11-24T02:47:15,677 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,677 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,677 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,677 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=985, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,678 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,678 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435670) roll requested 2024-11-24T02:47:15,678 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435678 2024-11-24T02:47:15,684 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,684 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435678 2024-11-24T02:47:15,685 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=986, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435678) roll requested 2024-11-24T02:47:15,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435686 2024-11-24T02:47:15,694 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,694 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,694 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,694 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,695 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,695 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435678 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435686 2024-11-24T02:47:15,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,696 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,696 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,696 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=987, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,696 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,696 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435686) roll requested 2024-11-24T02:47:15,696 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435696 2024-11-24T02:47:15,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435696 2024-11-24T02:47:15,705 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=988, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435696) roll requested 2024-11-24T02:47:15,706 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435706 2024-11-24T02:47:15,712 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,712 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,712 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,712 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,712 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,712 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435696 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435706 2024-11-24T02:47:15,713 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,713 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,713 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=989, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435706) roll requested 2024-11-24T02:47:15,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435714 2024-11-24T02:47:15,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,722 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,722 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435706 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435714 2024-11-24T02:47:15,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,723 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=990, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435714) roll requested 2024-11-24T02:47:15,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435724 2024-11-24T02:47:15,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435724 2024-11-24T02:47:15,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=991, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435724) roll requested 2024-11-24T02:47:15,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435731 2024-11-24T02:47:15,738 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,738 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,739 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,739 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,739 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,739 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435731 2024-11-24T02:47:15,740 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,740 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,740 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,741 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=992, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,741 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,741 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435731) roll requested 2024-11-24T02:47:15,741 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435741 2024-11-24T02:47:15,748 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,748 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,748 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,748 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,748 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,748 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435741 2024-11-24T02:47:15,749 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,749 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,749 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,750 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=993, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,750 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,750 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435741) roll requested 2024-11-24T02:47:15,750 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435750 2024-11-24T02:47:15,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,756 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,756 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,756 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,756 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435741 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435750 2024-11-24T02:47:15,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,757 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,757 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=994, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,757 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,757 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435750) roll requested 2024-11-24T02:47:15,757 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435757 2024-11-24T02:47:15,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435750 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435757 2024-11-24T02:47:15,766 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,766 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,766 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,767 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=995, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,767 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,767 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435757) roll requested 2024-11-24T02:47:15,767 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435767 2024-11-24T02:47:15,772 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,772 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,772 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,772 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,772 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,772 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435757 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435767 2024-11-24T02:47:15,773 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,773 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,773 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,773 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=996, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,773 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,773 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435767) roll requested 2024-11-24T02:47:15,773 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435773 2024-11-24T02:47:15,778 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,778 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,778 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,778 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,778 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,778 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435767 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435773 2024-11-24T02:47:15,779 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,779 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,779 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,779 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=997, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,779 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,779 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435773) roll requested 2024-11-24T02:47:15,779 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435779 2024-11-24T02:47:15,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435773 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435779 2024-11-24T02:47:15,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,789 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,789 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=998, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,789 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435779) roll requested 2024-11-24T02:47:15,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435789 2024-11-24T02:47:15,797 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,797 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,797 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,797 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,798 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,798 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435779 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435789 2024-11-24T02:47:15,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=999, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,799 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,799 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435789) roll requested 2024-11-24T02:47:15,799 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435799 2024-11-24T02:47:15,806 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,806 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,806 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,807 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,807 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,807 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435799 2024-11-24T02:47:15,807 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,808 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,808 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,808 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1000, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,808 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,808 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435799) roll requested 2024-11-24T02:47:15,808 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435808 2024-11-24T02:47:15,817 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,817 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,817 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,817 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,817 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,817 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435799 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435808 2024-11-24T02:47:15,818 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,818 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,818 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,818 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1001, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,818 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,818 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435808) roll requested 2024-11-24T02:47:15,819 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435819 2024-11-24T02:47:15,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,824 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435808 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435819 2024-11-24T02:47:15,825 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,825 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,825 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1002, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435819) roll requested 2024-11-24T02:47:15,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435825 2024-11-24T02:47:15,831 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,831 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,832 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,832 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,832 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,832 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435819 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435825 2024-11-24T02:47:15,833 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,833 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,833 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,833 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1003, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,833 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,833 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435825) roll requested 2024-11-24T02:47:15,833 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435833 2024-11-24T02:47:15,841 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,841 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,841 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,841 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,841 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,841 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435833 2024-11-24T02:47:15,842 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,842 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,842 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,842 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1004, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,842 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,842 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435833) roll requested 2024-11-24T02:47:15,843 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435842 2024-11-24T02:47:15,848 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,848 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,848 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,848 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,849 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,849 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435833 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435842 2024-11-24T02:47:15,849 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,849 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,849 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,850 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1005, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,850 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,850 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435842) roll requested 2024-11-24T02:47:15,850 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435850 2024-11-24T02:47:15,855 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,855 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,855 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,856 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,856 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,856 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435842 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435850 2024-11-24T02:47:15,857 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,857 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,857 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,857 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1006, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435850) roll requested 2024-11-24T02:47:15,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435858 2024-11-24T02:47:15,865 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,865 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,865 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,865 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,866 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,866 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435850 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435858 2024-11-24T02:47:15,866 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,866 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,866 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,867 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1007, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,867 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,867 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435858) roll requested 2024-11-24T02:47:15,867 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435867 2024-11-24T02:47:15,872 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,872 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,872 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,872 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,872 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,872 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435867 2024-11-24T02:47:15,873 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,873 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,873 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,874 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1008, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,874 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,874 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435867) roll requested 2024-11-24T02:47:15,874 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435874 2024-11-24T02:47:15,879 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,879 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435867 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435874 2024-11-24T02:47:15,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1009, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,881 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,881 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435874) roll requested 2024-11-24T02:47:15,881 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435881 2024-11-24T02:47:15,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,888 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435874 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435881 2024-11-24T02:47:15,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,889 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1010, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,889 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,889 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435881) roll requested 2024-11-24T02:47:15,889 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435889 2024-11-24T02:47:15,895 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,895 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,895 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,895 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,895 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,895 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435881 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435889 2024-11-24T02:47:15,896 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,896 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,896 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,897 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1011, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,897 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,897 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435889) roll requested 2024-11-24T02:47:15,897 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435897 2024-11-24T02:47:15,902 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,903 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,903 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,903 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,903 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,903 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435889 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435897 2024-11-24T02:47:15,904 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,904 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,904 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,904 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1012, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,904 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,904 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435897) roll requested 2024-11-24T02:47:15,904 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435904 2024-11-24T02:47:15,910 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,910 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,910 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,910 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,910 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,910 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435897 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435904 2024-11-24T02:47:15,911 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,911 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,911 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,911 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1013, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,911 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,911 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435904) roll requested 2024-11-24T02:47:15,912 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435911 2024-11-24T02:47:15,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435904 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435911 2024-11-24T02:47:15,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1014, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,920 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,920 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435911) roll requested 2024-11-24T02:47:15,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435920 2024-11-24T02:47:15,926 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,926 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,926 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,926 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,926 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,926 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435911 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435920 2024-11-24T02:47:15,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1015, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,928 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,928 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435920) roll requested 2024-11-24T02:47:15,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435928 2024-11-24T02:47:15,934 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,935 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435928 2024-11-24T02:47:15,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,936 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,936 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,936 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1016, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,936 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,936 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435928) roll requested 2024-11-24T02:47:15,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435937 2024-11-24T02:47:15,944 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,944 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,945 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,945 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,945 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,945 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435937 2024-11-24T02:47:15,946 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,946 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,946 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,946 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1017, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,946 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,946 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435937) roll requested 2024-11-24T02:47:15,947 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435947 2024-11-24T02:47:15,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435947 2024-11-24T02:47:15,956 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,956 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,956 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,956 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1018, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,956 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,956 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435947) roll requested 2024-11-24T02:47:15,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435957 2024-11-24T02:47:15,965 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,965 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,965 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435947 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435957 2024-11-24T02:47:15,967 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,967 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1019, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435957) roll requested 2024-11-24T02:47:15,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435967 2024-11-24T02:47:15,975 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,975 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,975 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,975 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,975 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,975 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435967 2024-11-24T02:47:15,976 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,976 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,976 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,977 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1020, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,977 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,977 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435967) roll requested 2024-11-24T02:47:15,977 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435977 2024-11-24T02:47:15,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435977 2024-11-24T02:47:15,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,986 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,986 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1021, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,986 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,986 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435977) roll requested 2024-11-24T02:47:15,986 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435986 2024-11-24T02:47:15,993 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,993 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,993 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,993 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,993 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:15,993 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435977 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435986 2024-11-24T02:47:15,994 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:15,994 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,994 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:15,994 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1022, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:15,994 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:15,994 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435986) roll requested 2024-11-24T02:47:15,994 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416435994 2024-11-24T02:47:16,000 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,000 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,000 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,000 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,000 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,000 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435986 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435994 2024-11-24T02:47:16,002 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,002 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,002 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,002 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1023, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,002 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,002 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416435994) roll requested 2024-11-24T02:47:16,002 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436002 2024-11-24T02:47:16,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,011 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,011 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,011 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416435994 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436002 2024-11-24T02:47:16,013 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,013 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,013 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,013 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1024, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,013 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,013 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436002) roll requested 2024-11-24T02:47:16,013 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436013 2024-11-24T02:47:16,024 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,024 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,024 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,025 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,025 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,025 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436002 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436013 2024-11-24T02:47:16,025 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,026 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,026 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,026 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1025, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,026 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,026 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436013) roll requested 2024-11-24T02:47:16,026 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436026 2024-11-24T02:47:16,032 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,032 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,032 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,032 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,032 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,032 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436013 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436026 2024-11-24T02:47:16,033 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,033 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1026, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,033 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,033 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,033 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,033 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436026) roll requested 2024-11-24T02:47:16,033 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436033 2024-11-24T02:47:16,039 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,039 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,039 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,039 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,039 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,039 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436026 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436033 2024-11-24T02:47:16,040 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,040 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,040 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,040 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1027, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,040 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,040 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436033) roll requested 2024-11-24T02:47:16,040 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436040 2024-11-24T02:47:16,045 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,045 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,045 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,045 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,045 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,045 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436033 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436040 2024-11-24T02:47:16,046 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,046 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,046 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,047 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1028, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,047 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,047 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436040) roll requested 2024-11-24T02:47:16,047 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436047 2024-11-24T02:47:16,053 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,053 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,053 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,053 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,053 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,053 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436040 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436047 2024-11-24T02:47:16,054 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,054 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,054 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,054 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1029, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,054 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,054 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436047) roll requested 2024-11-24T02:47:16,055 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436055 2024-11-24T02:47:16,060 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,060 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,060 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,060 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,060 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,060 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436047 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436055 2024-11-24T02:47:16,061 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,061 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,061 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,061 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1030, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,061 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,062 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436055) roll requested 2024-11-24T02:47:16,062 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436062 2024-11-24T02:47:16,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,068 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436055 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436062 2024-11-24T02:47:16,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1031, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436062) roll requested 2024-11-24T02:47:16,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436070 2024-11-24T02:47:16,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,079 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,079 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436062 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436070 2024-11-24T02:47:16,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,082 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,082 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1032, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,082 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,082 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436070) roll requested 2024-11-24T02:47:16,082 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436082 2024-11-24T02:47:16,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,094 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,094 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,094 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,094 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436082 2024-11-24T02:47:16,100 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,100 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,101 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,101 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1033, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,101 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,101 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436082) roll requested 2024-11-24T02:47:16,101 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436101 2024-11-24T02:47:16,107 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,107 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,107 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,107 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,107 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,107 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436082 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436101 2024-11-24T02:47:16,108 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,108 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,108 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,108 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1034, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,108 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,109 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436101) roll requested 2024-11-24T02:47:16,109 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436109 2024-11-24T02:47:16,114 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,114 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,114 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,114 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,114 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,114 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436101 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436109 2024-11-24T02:47:16,122 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,122 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,122 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,122 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1035, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,122 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,123 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436109) roll requested 2024-11-24T02:47:16,123 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436123 2024-11-24T02:47:16,132 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,132 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,132 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,132 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,132 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,132 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436109 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436123 2024-11-24T02:47:16,133 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,133 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,133 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,134 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1036, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,134 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,134 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436123) roll requested 2024-11-24T02:47:16,134 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436134 2024-11-24T02:47:16,146 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,146 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,147 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,147 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,147 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,147 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436123 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436134 2024-11-24T02:47:16,150 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,150 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,150 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,150 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1037, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,150 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,150 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436134) roll requested 2024-11-24T02:47:16,150 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436150 2024-11-24T02:47:16,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,156 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,156 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436134 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436150 2024-11-24T02:47:16,160 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,160 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,160 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,161 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1038, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,161 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,161 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436150) roll requested 2024-11-24T02:47:16,161 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436161 2024-11-24T02:47:16,170 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,170 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,170 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,170 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,170 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,170 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436150 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436161 2024-11-24T02:47:16,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,172 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1039, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436161) roll requested 2024-11-24T02:47:16,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436173 2024-11-24T02:47:16,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436161 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436173 2024-11-24T02:47:16,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1040, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436173) roll requested 2024-11-24T02:47:16,180 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436180 2024-11-24T02:47:16,186 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,186 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,186 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,186 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,186 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,186 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436180 2024-11-24T02:47:16,187 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,187 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,187 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,187 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1041, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,187 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,187 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436180) roll requested 2024-11-24T02:47:16,187 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436187 2024-11-24T02:47:16,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,194 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,194 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436187 2024-11-24T02:47:16,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1042, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,195 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,195 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436187) roll requested 2024-11-24T02:47:16,195 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436195 2024-11-24T02:47:16,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,201 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436187 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436195 2024-11-24T02:47:16,202 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,202 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,202 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,202 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1043, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,202 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,202 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436195) roll requested 2024-11-24T02:47:16,203 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436202 2024-11-24T02:47:16,208 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,208 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,208 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,208 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,208 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,209 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436195 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436202 2024-11-24T02:47:16,209 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,209 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,209 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,210 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1044, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,210 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,210 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436202) roll requested 2024-11-24T02:47:16,210 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436210 2024-11-24T02:47:16,215 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,215 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,215 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,215 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,215 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,215 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436202 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436210 2024-11-24T02:47:16,216 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,216 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,216 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,217 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1045, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,217 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,217 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436210) roll requested 2024-11-24T02:47:16,217 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436217 2024-11-24T02:47:16,221 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,221 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,221 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,221 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,221 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,222 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436210 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436217 2024-11-24T02:47:16,222 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,223 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,223 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,223 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1046, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,223 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,223 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436217) roll requested 2024-11-24T02:47:16,223 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436223 2024-11-24T02:47:16,228 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,228 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,229 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,229 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,229 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,229 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436217 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436223 2024-11-24T02:47:16,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1047, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,233 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436223) roll requested 2024-11-24T02:47:16,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436233 2024-11-24T02:47:16,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,238 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,238 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,238 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,238 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436223 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436233 2024-11-24T02:47:16,239 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,239 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,239 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,239 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1048, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,239 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,239 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436233) roll requested 2024-11-24T02:47:16,239 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436239 2024-11-24T02:47:16,244 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,244 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436239 2024-11-24T02:47:16,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,244 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1049, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436239) roll requested 2024-11-24T02:47:16,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436245 2024-11-24T02:47:16,250 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,250 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,250 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,250 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,250 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,250 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436239 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436245 2024-11-24T02:47:16,251 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,251 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,251 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,251 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1050, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,252 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436245) roll requested 2024-11-24T02:47:16,252 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436252 2024-11-24T02:47:16,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,257 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,257 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,257 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436252 2024-11-24T02:47:16,258 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,258 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,259 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,259 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1051, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,259 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,259 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436252) roll requested 2024-11-24T02:47:16,259 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436259 2024-11-24T02:47:16,264 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,264 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,265 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,265 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,265 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,265 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436252 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436259 2024-11-24T02:47:16,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,266 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,266 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1052, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,266 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,266 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436259) roll requested 2024-11-24T02:47:16,266 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436266 2024-11-24T02:47:16,271 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,271 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436259 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436266 2024-11-24T02:47:16,272 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,272 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,272 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,272 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1053, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,272 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,272 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436266) roll requested 2024-11-24T02:47:16,272 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436272 2024-11-24T02:47:16,277 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,277 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,277 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,277 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,277 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,277 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436266 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436272 2024-11-24T02:47:16,280 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,280 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,280 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1054, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,281 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,281 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436272) roll requested 2024-11-24T02:47:16,281 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436281 2024-11-24T02:47:16,291 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,291 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,291 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,291 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,292 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436272 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436281 2024-11-24T02:47:16,293 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,293 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,293 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,293 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1055, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,293 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,293 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436281) roll requested 2024-11-24T02:47:16,293 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436293 2024-11-24T02:47:16,303 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,303 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,303 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,303 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,303 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,303 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436281 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436293 2024-11-24T02:47:16,304 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,304 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,304 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,304 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1056, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,305 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,305 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436293) roll requested 2024-11-24T02:47:16,305 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436305 2024-11-24T02:47:16,312 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,312 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,313 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436293 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436305 2024-11-24T02:47:16,313 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,313 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,313 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,314 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1057, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,314 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,314 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436305) roll requested 2024-11-24T02:47:16,314 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436314 2024-11-24T02:47:16,319 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,319 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,320 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,320 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,320 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,320 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436305 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436314 2024-11-24T02:47:16,320 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,320 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,320 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,321 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1058, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,321 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,321 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436314) roll requested 2024-11-24T02:47:16,321 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436321 2024-11-24T02:47:16,327 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,327 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,327 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,327 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,327 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,327 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436314 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436321 2024-11-24T02:47:16,328 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,328 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,328 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,328 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1059, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,328 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,328 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436321) roll requested 2024-11-24T02:47:16,328 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436328 2024-11-24T02:47:16,333 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,333 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,334 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,334 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,334 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,334 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436321 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436328 2024-11-24T02:47:16,334 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,334 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,335 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,335 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1060, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,335 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,335 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436328) roll requested 2024-11-24T02:47:16,335 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436335 2024-11-24T02:47:16,341 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,341 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,341 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,341 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,341 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,341 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436328 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436335 2024-11-24T02:47:16,345 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,345 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,345 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,345 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1061, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,345 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,345 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436335) roll requested 2024-11-24T02:47:16,345 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436345 2024-11-24T02:47:16,351 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,352 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,352 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,352 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,352 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,352 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436335 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436345 2024-11-24T02:47:16,352 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,353 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,353 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,353 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1062, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,353 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,353 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436345) roll requested 2024-11-24T02:47:16,353 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436353 2024-11-24T02:47:16,359 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,359 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,359 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,359 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,359 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,359 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436345 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436353 2024-11-24T02:47:16,360 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,360 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,360 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,360 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1063, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,360 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,360 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436353) roll requested 2024-11-24T02:47:16,361 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436360 2024-11-24T02:47:16,366 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,366 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,366 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,366 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,366 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,366 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436353 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436360 2024-11-24T02:47:16,367 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,367 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,367 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,367 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1064, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,367 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,367 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436360) roll requested 2024-11-24T02:47:16,368 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436368 2024-11-24T02:47:16,373 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,373 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,374 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,374 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,374 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,374 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436360 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436368 2024-11-24T02:47:16,374 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,374 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,374 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,375 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1065, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,375 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,375 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436368) roll requested 2024-11-24T02:47:16,375 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436375 2024-11-24T02:47:16,381 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,381 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,381 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,381 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,381 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,381 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436368 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436375 2024-11-24T02:47:16,384 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,384 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,384 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,384 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1066, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,384 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,385 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436375) roll requested 2024-11-24T02:47:16,385 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436385 2024-11-24T02:47:16,393 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,393 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,393 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,393 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,393 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436375 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436385 2024-11-24T02:47:16,396 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,396 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,397 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,397 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1067, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,397 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,397 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436385) roll requested 2024-11-24T02:47:16,397 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436397 2024-11-24T02:47:16,403 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,403 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,403 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,403 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,403 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,403 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436385 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436397 2024-11-24T02:47:16,404 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,404 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,404 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,404 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1068, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,404 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,404 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436397) roll requested 2024-11-24T02:47:16,404 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436404 2024-11-24T02:47:16,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436397 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436404 2024-11-24T02:47:16,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1069, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,416 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436404) roll requested 2024-11-24T02:47:16,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436417 2024-11-24T02:47:16,421 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,421 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,421 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436404 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436417 2024-11-24T02:47:16,424 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,424 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,424 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,425 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1070, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,425 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,425 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436417) roll requested 2024-11-24T02:47:16,425 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436425 2024-11-24T02:47:16,432 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,433 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,433 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436425 2024-11-24T02:47:16,434 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,434 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,434 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,434 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1071, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,434 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,434 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436425) roll requested 2024-11-24T02:47:16,434 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436434 2024-11-24T02:47:16,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,445 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436425 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436434 2024-11-24T02:47:16,446 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,446 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,446 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,446 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1072, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,447 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,447 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436434) roll requested 2024-11-24T02:47:16,447 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436447 2024-11-24T02:47:16,459 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,459 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,459 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,459 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,459 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,459 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436434 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436447 2024-11-24T02:47:16,461 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,461 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,461 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,461 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1073, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436447) roll requested 2024-11-24T02:47:16,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436461 2024-11-24T02:47:16,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,470 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436447 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436461 2024-11-24T02:47:16,471 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,471 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,471 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,471 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1074, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,472 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,472 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436461) roll requested 2024-11-24T02:47:16,472 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436472 2024-11-24T02:47:16,476 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,476 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,477 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,477 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,477 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436472 2024-11-24T02:47:16,480 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,480 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,480 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,480 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1075, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,480 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,480 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436472) roll requested 2024-11-24T02:47:16,480 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436480 2024-11-24T02:47:16,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,485 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436472 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436480 2024-11-24T02:47:16,488 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,488 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,488 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,488 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1076, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,489 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,489 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436480) roll requested 2024-11-24T02:47:16,489 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436489 2024-11-24T02:47:16,494 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,494 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,494 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,494 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,494 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,494 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436480 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436489 2024-11-24T02:47:16,498 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,498 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,498 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,498 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1077, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,498 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,498 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436489) roll requested 2024-11-24T02:47:16,498 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436498 2024-11-24T02:47:16,503 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,503 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,503 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,503 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,503 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,503 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436489 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436498 2024-11-24T02:47:16,504 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,504 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,504 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,504 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1078, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,504 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,504 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436498) roll requested 2024-11-24T02:47:16,504 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436504 2024-11-24T02:47:16,509 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,509 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,509 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436498 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436504 2024-11-24T02:47:16,510 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,510 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,510 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,510 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1079, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,510 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,510 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436504) roll requested 2024-11-24T02:47:16,510 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436510 2024-11-24T02:47:16,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,514 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436504 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436510 2024-11-24T02:47:16,517 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,517 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,517 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,517 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1080, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,517 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,517 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436510) roll requested 2024-11-24T02:47:16,517 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436517 2024-11-24T02:47:16,522 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436510 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436517 2024-11-24T02:47:16,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1081, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436517) roll requested 2024-11-24T02:47:16,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436523 2024-11-24T02:47:16,532 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,532 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,532 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,532 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,532 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,532 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436517 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436523 2024-11-24T02:47:16,532 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,533 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,533 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,533 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1082, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,533 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,533 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436523) roll requested 2024-11-24T02:47:16,533 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436533 2024-11-24T02:47:16,538 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,538 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,538 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,538 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,538 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,538 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436533 2024-11-24T02:47:16,539 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,539 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,539 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,539 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1083, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,539 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,539 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436533) roll requested 2024-11-24T02:47:16,539 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436539 2024-11-24T02:47:16,544 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,544 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,544 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,544 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,545 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,545 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436533 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436539 2024-11-24T02:47:16,548 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,548 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,548 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,548 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1084, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,549 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,549 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436539) roll requested 2024-11-24T02:47:16,549 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436549 2024-11-24T02:47:16,554 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,554 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,554 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,554 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,554 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,554 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436539 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436549 2024-11-24T02:47:16,555 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,555 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,555 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,555 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1085, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,555 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,555 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436549) roll requested 2024-11-24T02:47:16,555 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436555 2024-11-24T02:47:16,562 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,562 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,563 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,563 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,563 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436549 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436555 2024-11-24T02:47:16,563 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,564 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,564 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,564 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1086, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,564 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,564 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436555) roll requested 2024-11-24T02:47:16,564 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436564 2024-11-24T02:47:16,569 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,569 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,569 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,569 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,569 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,569 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436555 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436564 2024-11-24T02:47:16,570 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,570 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,570 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,570 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1087, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,570 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,570 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436564) roll requested 2024-11-24T02:47:16,570 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436570 2024-11-24T02:47:16,575 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,575 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,575 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,575 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,575 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,575 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436564 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436570 2024-11-24T02:47:16,576 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,576 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,576 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,576 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1088, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,576 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,576 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436570) roll requested 2024-11-24T02:47:16,577 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436577 2024-11-24T02:47:16,581 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,581 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,581 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,581 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,581 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,581 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436570 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436577 2024-11-24T02:47:16,582 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,582 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,582 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,582 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1089, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,583 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,583 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436577) roll requested 2024-11-24T02:47:16,583 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436583 2024-11-24T02:47:16,587 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,587 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,587 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,587 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436577 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436583 2024-11-24T02:47:16,588 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,588 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,588 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,589 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1090, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,589 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,589 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436583) roll requested 2024-11-24T02:47:16,589 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436589 2024-11-24T02:47:16,593 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,593 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,593 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,593 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,593 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,594 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436583 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436589 2024-11-24T02:47:16,594 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,595 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,595 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,595 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1091, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,595 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,595 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436589) roll requested 2024-11-24T02:47:16,595 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436595 2024-11-24T02:47:16,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,599 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,599 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436589 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436595 2024-11-24T02:47:16,600 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,600 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,600 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,601 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1092, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,601 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,601 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436595) roll requested 2024-11-24T02:47:16,601 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436601 2024-11-24T02:47:16,605 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,605 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,606 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,606 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,606 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,606 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436595 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436601 2024-11-24T02:47:16,607 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,607 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,607 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,607 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1093, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,607 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,607 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436601) roll requested 2024-11-24T02:47:16,607 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436607 2024-11-24T02:47:16,611 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,611 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,611 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,611 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,612 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,612 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436601 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436607 2024-11-24T02:47:16,612 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,612 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,612 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,613 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1094, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,613 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,613 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436607) roll requested 2024-11-24T02:47:16,613 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436613 2024-11-24T02:47:16,617 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,617 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,617 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,617 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,617 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,617 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436607 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436613 2024-11-24T02:47:16,618 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,618 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,618 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,619 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1095, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,619 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,619 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436613) roll requested 2024-11-24T02:47:16,619 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436619 2024-11-24T02:47:16,623 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,623 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,623 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,623 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,623 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,623 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436613 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436619 2024-11-24T02:47:16,624 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,624 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,624 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,624 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1096, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,624 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,624 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436619) roll requested 2024-11-24T02:47:16,625 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436624 2024-11-24T02:47:16,629 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,629 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,629 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,629 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,629 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,629 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436619 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436624 2024-11-24T02:47:16,630 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,630 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,630 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,630 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1097, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,630 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,630 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436624) roll requested 2024-11-24T02:47:16,630 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436630 2024-11-24T02:47:16,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,635 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436624 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436630 2024-11-24T02:47:16,635 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,635 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,635 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,635 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1098, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,635 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,635 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436630) roll requested 2024-11-24T02:47:16,635 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436635 2024-11-24T02:47:16,639 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,640 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,640 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,640 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,640 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,640 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436630 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436635 2024-11-24T02:47:16,640 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,641 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,641 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,641 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1099, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,641 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,641 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436635) roll requested 2024-11-24T02:47:16,641 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436641 2024-11-24T02:47:16,645 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,646 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,646 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,646 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,646 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,646 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436635 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436641 2024-11-24T02:47:16,646 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,646 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,647 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,647 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1100, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,647 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,647 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436641) roll requested 2024-11-24T02:47:16,647 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436647 2024-11-24T02:47:16,651 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,651 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,651 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,651 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,652 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436641 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436647 2024-11-24T02:47:16,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,653 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1101, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,653 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,653 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436647) roll requested 2024-11-24T02:47:16,653 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436653 2024-11-24T02:47:16,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436647 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436653 2024-11-24T02:47:16,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1102, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436653) roll requested 2024-11-24T02:47:16,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436658 2024-11-24T02:47:16,662 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,662 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,662 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,662 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,662 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436653 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436658 2024-11-24T02:47:16,663 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,663 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,663 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1103, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436658) roll requested 2024-11-24T02:47:16,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436664 2024-11-24T02:47:16,668 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,668 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436664 2024-11-24T02:47:16,669 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,669 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,669 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,669 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1104, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436664) roll requested 2024-11-24T02:47:16,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436670 2024-11-24T02:47:16,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,674 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,674 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,674 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436670 2024-11-24T02:47:16,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1105, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,675 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,675 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436670) roll requested 2024-11-24T02:47:16,675 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436675 2024-11-24T02:47:16,679 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,679 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,679 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,679 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436675 2024-11-24T02:47:16,680 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,680 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,680 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,680 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1106, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,680 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436675) roll requested 2024-11-24T02:47:16,681 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436681 2024-11-24T02:47:16,685 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,685 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,685 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,685 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,685 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,685 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436675 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436681 2024-11-24T02:47:16,686 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,686 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,686 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,686 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1107, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,686 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,686 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436681) roll requested 2024-11-24T02:47:16,686 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436686 2024-11-24T02:47:16,690 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,690 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,690 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,690 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,690 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,690 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436686 2024-11-24T02:47:16,691 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,691 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,691 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,691 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1108, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,691 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,691 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436686) roll requested 2024-11-24T02:47:16,692 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436691 2024-11-24T02:47:16,696 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,696 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,696 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,696 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,696 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,696 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436686 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436691 2024-11-24T02:47:16,696 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,697 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,697 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,697 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1109, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,697 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,697 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436691) roll requested 2024-11-24T02:47:16,697 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436697 2024-11-24T02:47:16,701 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,701 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,701 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,701 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,701 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,701 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436691 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436697 2024-11-24T02:47:16,702 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,702 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,702 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,702 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1110, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,702 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,702 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436697) roll requested 2024-11-24T02:47:16,702 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436702 2024-11-24T02:47:16,707 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,707 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,707 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,707 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,707 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,707 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436697 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436702 2024-11-24T02:47:16,708 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,708 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,708 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,708 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1111, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,708 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,708 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436702) roll requested 2024-11-24T02:47:16,709 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436709 2024-11-24T02:47:16,713 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,713 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,713 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,713 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,713 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,713 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436702 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436709 2024-11-24T02:47:16,714 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,714 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,714 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,714 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1112, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,714 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,714 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436709) roll requested 2024-11-24T02:47:16,714 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436714 2024-11-24T02:47:16,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,719 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,719 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,719 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436709 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436714 2024-11-24T02:47:16,719 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,719 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,719 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,720 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1113, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,720 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,720 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436714) roll requested 2024-11-24T02:47:16,720 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436720 2024-11-24T02:47:16,724 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,724 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,724 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,724 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,724 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,724 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436714 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436720 2024-11-24T02:47:16,725 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,725 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,725 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,725 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1114, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,725 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,725 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436720) roll requested 2024-11-24T02:47:16,725 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436725 2024-11-24T02:47:16,730 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,730 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,730 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,730 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,730 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,730 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436720 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436725 2024-11-24T02:47:16,730 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,731 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,731 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,731 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1115, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,731 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,731 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436725) roll requested 2024-11-24T02:47:16,731 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436731 2024-11-24T02:47:16,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436725 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436731 2024-11-24T02:47:16,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,737 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1116, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,737 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,737 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436731) roll requested 2024-11-24T02:47:16,737 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436737 2024-11-24T02:47:16,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436731 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436737 2024-11-24T02:47:16,742 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,742 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,742 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,742 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1117, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,742 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,742 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436737) roll requested 2024-11-24T02:47:16,742 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436742 2024-11-24T02:47:16,746 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,746 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,746 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,746 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,747 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,747 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436737 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436742 2024-11-24T02:47:16,747 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,747 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,747 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,748 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1118, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,748 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,748 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436742) roll requested 2024-11-24T02:47:16,748 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436748 2024-11-24T02:47:16,752 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,752 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,752 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,753 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,753 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,753 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436742 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436748 2024-11-24T02:47:16,753 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,753 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,753 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,753 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1119, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,753 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,753 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436748) roll requested 2024-11-24T02:47:16,754 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436754 2024-11-24T02:47:16,758 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,758 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,758 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,758 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,758 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,758 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436748 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436754 2024-11-24T02:47:16,759 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,759 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,759 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,759 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1120, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,759 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,759 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436754) roll requested 2024-11-24T02:47:16,759 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436759 2024-11-24T02:47:16,763 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,763 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,764 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,764 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,764 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,764 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436754 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436759 2024-11-24T02:47:16,764 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,765 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,765 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,765 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1121, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,765 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,765 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436759) roll requested 2024-11-24T02:47:16,765 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436765 2024-11-24T02:47:16,769 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,769 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,769 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,769 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,769 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,770 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436759 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436765 2024-11-24T02:47:16,770 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,770 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,770 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,770 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1122, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,770 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,770 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436765) roll requested 2024-11-24T02:47:16,771 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436770 2024-11-24T02:47:16,775 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,775 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,775 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,775 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,775 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,775 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436765 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436770 2024-11-24T02:47:16,776 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,776 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,776 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,776 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1123, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,776 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,777 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436770) roll requested 2024-11-24T02:47:16,777 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436777 2024-11-24T02:47:16,781 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,781 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,781 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,781 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,781 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,781 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436770 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436777 2024-11-24T02:47:16,782 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,782 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,782 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,782 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1124, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,782 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,782 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436777) roll requested 2024-11-24T02:47:16,782 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436782 2024-11-24T02:47:16,787 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,787 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,787 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,787 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,787 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,787 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436777 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436782 2024-11-24T02:47:16,788 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,788 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1125, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,788 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,788 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,788 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,789 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436782) roll requested 2024-11-24T02:47:16,789 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436789 2024-11-24T02:47:16,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,793 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,793 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,793 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,793 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,793 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436782 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436789 2024-11-24T02:47:16,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,794 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,794 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1126, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,794 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,794 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436789) roll requested 2024-11-24T02:47:16,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436794 2024-11-24T02:47:16,798 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,798 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,798 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,798 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,799 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,799 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436789 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436794 2024-11-24T02:47:16,799 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,799 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,799 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,799 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1127, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,800 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,800 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436794) roll requested 2024-11-24T02:47:16,800 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436800 2024-11-24T02:47:16,804 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,804 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,804 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,804 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,804 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,804 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436794 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436800 2024-11-24T02:47:16,805 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,805 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,805 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1128, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436800) roll requested 2024-11-24T02:47:16,806 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436806 2024-11-24T02:47:16,810 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,810 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,810 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,810 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,810 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,810 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436800 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436806 2024-11-24T02:47:16,811 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,811 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,811 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,811 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1129, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,811 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,811 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436806) roll requested 2024-11-24T02:47:16,811 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436811 2024-11-24T02:47:16,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,815 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,815 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,815 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436806 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436811 2024-11-24T02:47:16,816 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,816 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,816 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,817 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1130, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,817 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,817 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436811) roll requested 2024-11-24T02:47:16,817 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436817 2024-11-24T02:47:16,821 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,821 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,821 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,821 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,822 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,822 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436811 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436817 2024-11-24T02:47:16,822 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,822 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,823 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,823 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1131, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,823 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,823 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436817) roll requested 2024-11-24T02:47:16,823 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436823 2024-11-24T02:47:16,828 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,828 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,828 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,828 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,828 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,828 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436817 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436823 2024-11-24T02:47:16,829 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,829 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,829 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,829 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1132, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,829 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,829 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436823) roll requested 2024-11-24T02:47:16,829 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436829 2024-11-24T02:47:16,835 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,835 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436823 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436829 2024-11-24T02:47:16,836 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,836 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,836 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,836 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1133, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,836 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,836 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436829) roll requested 2024-11-24T02:47:16,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436836 2024-11-24T02:47:16,851 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,851 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,851 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,851 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,851 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,851 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436829 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436836 2024-11-24T02:47:16,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,853 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1134, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,853 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,853 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436836) roll requested 2024-11-24T02:47:16,853 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436853 2024-11-24T02:47:16,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,858 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,858 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436836 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436853 2024-11-24T02:47:16,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1135, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,859 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,859 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436853) roll requested 2024-11-24T02:47:16,859 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436859 2024-11-24T02:47:16,864 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,864 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,864 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,864 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,864 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,864 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436853 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436859 2024-11-24T02:47:16,865 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,865 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,865 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1136, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,866 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,866 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436859) roll requested 2024-11-24T02:47:16,866 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436866 2024-11-24T02:47:16,871 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,871 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,871 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,871 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,871 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,871 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436859 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436866 2024-11-24T02:47:16,872 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,872 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,872 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,873 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1137, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,873 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,873 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436866) roll requested 2024-11-24T02:47:16,873 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436873 2024-11-24T02:47:16,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,881 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,881 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436866 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436873 2024-11-24T02:47:16,881 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,881 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,881 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,881 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1138, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,882 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,882 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436873) roll requested 2024-11-24T02:47:16,882 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436882 2024-11-24T02:47:16,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,889 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,889 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436873 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436882 2024-11-24T02:47:16,889 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,889 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,889 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,890 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1139, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,890 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,890 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436882) roll requested 2024-11-24T02:47:16,890 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436890 2024-11-24T02:47:16,894 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,894 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,894 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,894 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,894 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,894 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436882 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436890 2024-11-24T02:47:16,895 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,895 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,895 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,895 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1140, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,895 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,896 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436890) roll requested 2024-11-24T02:47:16,896 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436896 2024-11-24T02:47:16,900 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,900 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,900 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,900 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,900 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,900 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436890 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436896 2024-11-24T02:47:16,901 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1141, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436896) roll requested 2024-11-24T02:47:16,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436901 2024-11-24T02:47:16,906 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,906 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,906 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,906 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,906 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436896 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436901 2024-11-24T02:47:16,907 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,907 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,907 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,907 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1142, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,908 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,908 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436901) roll requested 2024-11-24T02:47:16,908 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436908 2024-11-24T02:47:16,913 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,913 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,913 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,913 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,913 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,913 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436908 2024-11-24T02:47:16,913 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,913 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,914 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,914 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1143, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,914 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,914 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436908) roll requested 2024-11-24T02:47:16,914 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436914 2024-11-24T02:47:16,918 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,918 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,918 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,918 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,918 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,918 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436908 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436914 2024-11-24T02:47:16,919 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,919 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,919 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,919 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1144, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,919 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,919 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436914) roll requested 2024-11-24T02:47:16,920 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436920 2024-11-24T02:47:16,924 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,924 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,924 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,924 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,924 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,924 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436914 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436920 2024-11-24T02:47:16,925 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,925 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,925 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,925 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1145, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,925 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,925 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436920) roll requested 2024-11-24T02:47:16,925 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436925 2024-11-24T02:47:16,930 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,930 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,930 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,930 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,930 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,930 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436920 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436925 2024-11-24T02:47:16,931 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,931 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,931 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,931 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1146, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,931 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,931 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436925) roll requested 2024-11-24T02:47:16,931 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436931 2024-11-24T02:47:16,936 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,936 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,936 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,936 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,936 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436925 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436931 2024-11-24T02:47:16,936 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,937 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1147, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,937 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,937 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436931) roll requested 2024-11-24T02:47:16,937 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436937 2024-11-24T02:47:16,942 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,942 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,942 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,942 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,942 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,942 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436931 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436937 2024-11-24T02:47:16,943 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,943 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,943 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,943 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1148, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,943 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,943 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436937) roll requested 2024-11-24T02:47:16,944 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436943 2024-11-24T02:47:16,948 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,948 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,948 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,948 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,948 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,948 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436937 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436943 2024-11-24T02:47:16,949 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,949 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,949 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,949 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1149, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,949 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,949 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436943) roll requested 2024-11-24T02:47:16,950 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436949 2024-11-24T02:47:16,954 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,954 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,954 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,954 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,954 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,954 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436943 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436949 2024-11-24T02:47:16,955 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,955 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,955 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,955 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1150, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,955 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,955 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436949) roll requested 2024-11-24T02:47:16,955 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436955 2024-11-24T02:47:16,960 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,960 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,960 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,960 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,960 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,960 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436955 2024-11-24T02:47:16,961 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,961 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,961 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,961 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1151, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,961 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,961 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436955) roll requested 2024-11-24T02:47:16,961 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436961 2024-11-24T02:47:16,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,966 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,966 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,966 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,966 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,966 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436955 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436961 2024-11-24T02:47:16,966 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,966 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,967 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,967 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1152, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,967 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,967 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436961) roll requested 2024-11-24T02:47:16,967 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436967 2024-11-24T02:47:16,971 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,971 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,971 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,971 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,972 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,972 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436961 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436967 2024-11-24T02:47:16,972 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,973 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,973 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,973 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1153, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,973 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,973 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436967) roll requested 2024-11-24T02:47:16,973 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436973 2024-11-24T02:47:16,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,978 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,978 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,978 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,978 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436967 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436973 2024-11-24T02:47:16,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1154, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436973) roll requested 2024-11-24T02:47:16,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436979 2024-11-24T02:47:16,984 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,984 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,984 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,984 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436973 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436979 2024-11-24T02:47:16,985 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,985 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,985 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,985 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1155, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,985 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,985 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436979) roll requested 2024-11-24T02:47:16,985 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436985 2024-11-24T02:47:16,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,991 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,991 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,991 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,991 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436979 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436985 2024-11-24T02:47:16,992 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,992 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,993 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,993 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1156, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,993 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,993 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436985) roll requested 2024-11-24T02:47:16,993 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436993 2024-11-24T02:47:16,997 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,998 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,998 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,998 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,998 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:16,998 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436985 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436993 2024-11-24T02:47:16,999 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:16,999 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,999 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:16,999 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1157, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:16,999 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:16,999 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436993) roll requested 2024-11-24T02:47:16,999 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416436999 2024-11-24T02:47:17,004 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,004 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,004 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,004 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,004 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,004 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436993 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436999 2024-11-24T02:47:17,005 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,005 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,005 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,005 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1158, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,005 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,005 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416436999) roll requested 2024-11-24T02:47:17,005 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437005 2024-11-24T02:47:17,010 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,010 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,010 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,010 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,010 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,010 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416436999 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437005 2024-11-24T02:47:17,011 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,011 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,011 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,012 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1159, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,012 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,012 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437005) roll requested 2024-11-24T02:47:17,012 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437012 2024-11-24T02:47:17,016 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,016 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,017 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,017 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437005 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437012 2024-11-24T02:47:17,018 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,018 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,018 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,018 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1160, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,018 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,018 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437012) roll requested 2024-11-24T02:47:17,018 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437018 2024-11-24T02:47:17,023 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,023 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,023 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,023 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,024 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,024 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437012 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437018 2024-11-24T02:47:17,024 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,024 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,025 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,025 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1161, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,025 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,025 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437018) roll requested 2024-11-24T02:47:17,025 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437025 2024-11-24T02:47:17,030 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,030 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,030 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,030 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,030 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,030 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437018 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437025 2024-11-24T02:47:17,031 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,031 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,031 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,031 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1162, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,031 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,031 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437025) roll requested 2024-11-24T02:47:17,031 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437031 2024-11-24T02:47:17,036 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,036 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,036 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,037 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,037 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,037 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437025 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437031 2024-11-24T02:47:17,037 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,037 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,037 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,038 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1163, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,038 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,038 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437031) roll requested 2024-11-24T02:47:17,038 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437038 2024-11-24T02:47:17,043 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,043 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,043 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,043 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,043 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,043 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437031 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437038 2024-11-24T02:47:17,044 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,044 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,044 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,044 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1164, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,044 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,044 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437038) roll requested 2024-11-24T02:47:17,044 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437044 2024-11-24T02:47:17,049 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,049 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,049 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,049 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,049 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,049 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437038 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437044 2024-11-24T02:47:17,050 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,050 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,050 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,050 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1165, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,050 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,050 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437044) roll requested 2024-11-24T02:47:17,050 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437050 2024-11-24T02:47:17,055 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,055 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,055 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,055 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,055 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,055 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437044 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437050 2024-11-24T02:47:17,056 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,056 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,056 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,056 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1166, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,056 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,056 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437050) roll requested 2024-11-24T02:47:17,057 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437057 2024-11-24T02:47:17,063 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,063 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,063 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,063 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,063 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,063 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437050 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437057 2024-11-24T02:47:17,064 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,064 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,064 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,064 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1167, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,064 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,064 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437057) roll requested 2024-11-24T02:47:17,064 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437064 2024-11-24T02:47:17,069 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,069 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,069 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,069 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,069 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437057 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437064 2024-11-24T02:47:17,069 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,069 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,070 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,070 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1168, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,070 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,070 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437064) roll requested 2024-11-24T02:47:17,070 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437070 2024-11-24T02:47:17,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,074 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,074 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,074 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437064 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437070 2024-11-24T02:47:17,075 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,075 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,075 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,075 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1169, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,075 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,075 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437070) roll requested 2024-11-24T02:47:17,075 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437075 2024-11-24T02:47:17,080 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,080 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,080 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,080 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,080 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437070 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437075 2024-11-24T02:47:17,081 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,081 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,081 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,081 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1170, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,081 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,081 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437075) roll requested 2024-11-24T02:47:17,081 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437081 2024-11-24T02:47:17,086 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,086 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,086 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,086 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,087 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,087 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437075 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437081 2024-11-24T02:47:17,087 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,087 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,087 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,087 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1171, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,087 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,088 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437081) roll requested 2024-11-24T02:47:17,088 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437088 2024-11-24T02:47:17,094 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,094 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,095 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,095 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,095 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,095 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437081 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437088 2024-11-24T02:47:17,096 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,096 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,096 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,096 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1172, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,096 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,096 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437088) roll requested 2024-11-24T02:47:17,096 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437096 2024-11-24T02:47:17,102 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,102 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,102 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,102 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,102 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,102 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437088 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437096 2024-11-24T02:47:17,103 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,103 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,103 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,104 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1173, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,104 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,104 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437096) roll requested 2024-11-24T02:47:17,104 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437104 2024-11-24T02:47:17,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,111 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,111 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,111 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437096 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437104 2024-11-24T02:47:17,111 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,112 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,112 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,112 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1174, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,112 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,112 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437104) roll requested 2024-11-24T02:47:17,112 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437112 2024-11-24T02:47:17,120 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,120 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,120 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,120 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,120 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,120 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437104 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437112 2024-11-24T02:47:17,120 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,120 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,121 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,121 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1175, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,121 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,121 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437112) roll requested 2024-11-24T02:47:17,121 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437121 2024-11-24T02:47:17,126 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,126 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,127 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,127 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,127 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,127 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437112 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437121 2024-11-24T02:47:17,127 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,128 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,128 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,128 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1176, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,128 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,128 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437121) roll requested 2024-11-24T02:47:17,128 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437128 2024-11-24T02:47:17,134 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,134 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,134 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,134 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,134 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,134 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437121 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437128 2024-11-24T02:47:17,135 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,135 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,135 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,135 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1177, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,136 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,136 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437128) roll requested 2024-11-24T02:47:17,136 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437136 2024-11-24T02:47:17,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,143 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,143 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,143 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,143 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437128 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437136 2024-11-24T02:47:17,144 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,144 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,144 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,144 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1178, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,145 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,145 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437136) roll requested 2024-11-24T02:47:17,145 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437145 2024-11-24T02:47:17,152 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,152 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,152 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,152 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,152 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,152 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437136 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437145 2024-11-24T02:47:17,153 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,153 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,153 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,153 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1179, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,154 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,154 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437145) roll requested 2024-11-24T02:47:17,154 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437154 2024-11-24T02:47:17,161 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,161 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,161 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,161 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,162 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,162 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437145 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437154 2024-11-24T02:47:17,163 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,163 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,163 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,163 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1180, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,163 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,163 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437154) roll requested 2024-11-24T02:47:17,164 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437163 2024-11-24T02:47:17,171 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,171 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,171 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,171 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,171 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,172 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437154 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437163 2024-11-24T02:47:17,172 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,172 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,173 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,173 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1181, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,173 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,173 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437163) roll requested 2024-11-24T02:47:17,173 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437173 2024-11-24T02:47:17,179 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,179 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,179 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,179 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,179 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,179 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437163 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437173 2024-11-24T02:47:17,180 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,180 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,180 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,180 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1182, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,180 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,180 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437173) roll requested 2024-11-24T02:47:17,181 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437180 2024-11-24T02:47:17,187 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,187 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,187 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,187 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,187 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,187 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437173 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437180 2024-11-24T02:47:17,188 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,188 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,188 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,188 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1183, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,188 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,188 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437180) roll requested 2024-11-24T02:47:17,189 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437189 2024-11-24T02:47:17,193 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,193 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,193 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,193 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,193 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,193 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437180 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437189 2024-11-24T02:47:17,195 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,195 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,195 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,195 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1184, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,196 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,196 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437189) roll requested 2024-11-24T02:47:17,196 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437196 2024-11-24T02:47:17,202 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,202 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,202 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,202 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,203 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437189 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437196 2024-11-24T02:47:17,207 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,207 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,207 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,207 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1185, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,207 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,207 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437196) roll requested 2024-11-24T02:47:17,207 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437207 2024-11-24T02:47:17,212 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,212 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,212 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,212 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,212 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,212 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437196 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437207 2024-11-24T02:47:17,213 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,213 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,213 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,213 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1186, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,213 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,213 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437207) roll requested 2024-11-24T02:47:17,214 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437213 2024-11-24T02:47:17,218 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,218 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,218 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,218 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,218 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,219 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437207 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437213 2024-11-24T02:47:17,219 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,219 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,219 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,219 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1187, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,219 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,220 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437213) roll requested 2024-11-24T02:47:17,220 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437220 2024-11-24T02:47:17,224 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,224 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,224 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,224 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,224 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,224 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437213 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437220 2024-11-24T02:47:17,225 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,225 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,225 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,225 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1188, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,225 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,225 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437220) roll requested 2024-11-24T02:47:17,226 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437225 2024-11-24T02:47:17,230 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,230 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,230 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,230 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,231 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,231 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437220 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437225 2024-11-24T02:47:17,232 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,232 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,232 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,232 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1189, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,232 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,232 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437225) roll requested 2024-11-24T02:47:17,233 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437233 2024-11-24T02:47:17,237 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,237 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,237 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,237 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,237 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,238 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437225 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437233 2024-11-24T02:47:17,238 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,238 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,238 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,238 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1190, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,238 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,238 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437233) roll requested 2024-11-24T02:47:17,238 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437238 2024-11-24T02:47:17,243 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,243 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,244 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,244 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,244 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,244 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437233 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437238 2024-11-24T02:47:17,244 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,245 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,245 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,245 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1191, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,245 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,245 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437238) roll requested 2024-11-24T02:47:17,245 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437245 2024-11-24T02:47:17,249 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,249 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,249 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,249 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,249 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,249 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437238 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437245 2024-11-24T02:47:17,252 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,252 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,252 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,252 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1192, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,252 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,253 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437245) roll requested 2024-11-24T02:47:17,253 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437253 2024-11-24T02:47:17,257 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,257 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,257 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,258 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,258 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,258 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437245 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437253 2024-11-24T02:47:17,260 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,260 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,260 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,260 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1193, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,261 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,261 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437253) roll requested 2024-11-24T02:47:17,261 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437261 2024-11-24T02:47:17,265 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,266 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437253 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437261 2024-11-24T02:47:17,266 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,266 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,267 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,267 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1194, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,267 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,267 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437261) roll requested 2024-11-24T02:47:17,267 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437267 2024-11-24T02:47:17,272 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,272 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,272 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,273 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,273 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,273 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437261 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437267 2024-11-24T02:47:17,273 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,273 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,273 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,273 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1195, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,274 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,274 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437267) roll requested 2024-11-24T02:47:17,274 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437274 2024-11-24T02:47:17,279 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,279 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,279 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,279 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,279 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,279 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437267 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437274 2024-11-24T02:47:17,281 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,281 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,281 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,281 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1196, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,282 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,282 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437274) roll requested 2024-11-24T02:47:17,282 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437282 2024-11-24T02:47:17,287 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,287 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,287 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,287 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,287 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,287 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437274 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437282 2024-11-24T02:47:17,287 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,287 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,287 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,288 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1197, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,288 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,288 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437282) roll requested 2024-11-24T02:47:17,288 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437288 2024-11-24T02:47:17,292 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,292 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,292 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,292 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,292 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,293 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437282 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437288 2024-11-24T02:47:17,296 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,296 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,296 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,296 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1198, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,296 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,297 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437288) roll requested 2024-11-24T02:47:17,297 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437297 2024-11-24T02:47:17,302 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,302 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,302 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,302 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,302 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,302 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437288 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437297 2024-11-24T02:47:17,303 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,303 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,303 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,303 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1199, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,303 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,303 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437297) roll requested 2024-11-24T02:47:17,303 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437303 2024-11-24T02:47:17,308 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,308 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,308 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,308 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,308 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,308 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437297 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437303 2024-11-24T02:47:17,309 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,309 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,309 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,309 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1200, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,309 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,309 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437303) roll requested 2024-11-24T02:47:17,309 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437309 2024-11-24T02:47:17,314 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,314 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,314 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,314 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,314 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,315 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437303 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437309 2024-11-24T02:47:17,315 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,315 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,315 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,316 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1201, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,316 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,316 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437309) roll requested 2024-11-24T02:47:17,316 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437316 2024-11-24T02:47:17,320 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,320 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,321 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,321 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,321 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,321 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437309 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437316 2024-11-24T02:47:17,322 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,322 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,322 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,322 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1202, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,322 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,322 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437316) roll requested 2024-11-24T02:47:17,322 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437322 2024-11-24T02:47:17,392 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,392 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,392 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,392 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,392 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,393 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437316 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437322 2024-11-24T02:47:17,395 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,395 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,395 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,395 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1203, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,395 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,395 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437322) roll requested 2024-11-24T02:47:17,396 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437395 2024-11-24T02:47:17,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,401 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437322 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437395 2024-11-24T02:47:17,402 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,402 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,402 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,402 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1204, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,402 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,402 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437395) roll requested 2024-11-24T02:47:17,403 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437402 2024-11-24T02:47:17,408 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,409 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,409 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,409 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,409 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,409 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437395 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437402 2024-11-24T02:47:17,410 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,410 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,410 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,410 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1205, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,410 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,410 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437402) roll requested 2024-11-24T02:47:17,410 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437410 2024-11-24T02:47:17,415 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,415 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,415 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,415 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,415 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,415 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437402 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437410 2024-11-24T02:47:17,416 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,416 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,416 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,416 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1206, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,417 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,417 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437410) roll requested 2024-11-24T02:47:17,417 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437417 2024-11-24T02:47:17,422 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,422 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,422 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,422 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,422 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,422 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437410 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437417 2024-11-24T02:47:17,423 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,423 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,423 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,423 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1207, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,423 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,423 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437417) roll requested 2024-11-24T02:47:17,423 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437423 2024-11-24T02:47:17,427 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,427 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,427 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,427 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,427 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,427 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437417 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437423 2024-11-24T02:47:17,428 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,428 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,428 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,428 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1208, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,428 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,428 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437423) roll requested 2024-11-24T02:47:17,428 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437428 2024-11-24T02:47:17,433 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,433 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,433 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,433 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,434 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,434 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437423 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437428 2024-11-24T02:47:17,435 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,435 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,435 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,436 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1209, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,436 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,436 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437428) roll requested 2024-11-24T02:47:17,436 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437436 2024-11-24T02:47:17,442 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,442 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,442 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,442 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,442 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,443 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437428 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437436 2024-11-24T02:47:17,443 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,443 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,443 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,443 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1210, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,444 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,444 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437436) roll requested 2024-11-24T02:47:17,444 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437444 2024-11-24T02:47:17,448 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,448 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,448 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,448 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,448 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,449 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437436 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437444 2024-11-24T02:47:17,452 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,452 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,452 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,452 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1211, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,453 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,453 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437444) roll requested 2024-11-24T02:47:17,453 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437453 2024-11-24T02:47:17,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,457 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,457 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,457 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437444 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437453 2024-11-24T02:47:17,460 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,460 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,460 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,460 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1212, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,461 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,461 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437453) roll requested 2024-11-24T02:47:17,461 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437461 2024-11-24T02:47:17,465 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,465 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,465 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,465 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,465 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,466 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437453 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437461 2024-11-24T02:47:17,466 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,466 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,466 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,467 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1213, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,467 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,467 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437461) roll requested 2024-11-24T02:47:17,467 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437467 2024-11-24T02:47:17,472 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,472 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,472 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,472 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,472 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,472 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437461 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437467 2024-11-24T02:47:17,472 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,472 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,473 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,473 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1214, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,473 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,473 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437467) roll requested 2024-11-24T02:47:17,473 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437473 2024-11-24T02:47:17,477 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,477 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,477 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,478 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,478 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437467 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437473 2024-11-24T02:47:17,479 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,479 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,479 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,479 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1215, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,479 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,479 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437473) roll requested 2024-11-24T02:47:17,479 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437479 2024-11-24T02:47:17,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,484 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437473 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437479 2024-11-24T02:47:17,485 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,485 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,485 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,485 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1216, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,485 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,485 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437479) roll requested 2024-11-24T02:47:17,486 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437485 2024-11-24T02:47:17,490 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,490 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,490 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,490 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,490 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,491 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437479 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437485 2024-11-24T02:47:17,491 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,491 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,491 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,492 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1217, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,492 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,492 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437485) roll requested 2024-11-24T02:47:17,492 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437492 2024-11-24T02:47:17,496 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,496 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,496 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,496 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,496 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,496 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437485 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437492 2024-11-24T02:47:17,497 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,497 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,497 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,497 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1218, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,497 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,497 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437492) roll requested 2024-11-24T02:47:17,497 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437497 2024-11-24T02:47:17,501 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,502 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,502 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,502 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,502 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,502 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437492 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437497 2024-11-24T02:47:17,502 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,502 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,502 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,503 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1219, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,503 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,503 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437497) roll requested 2024-11-24T02:47:17,503 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437503 2024-11-24T02:47:17,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,508 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,508 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,508 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,508 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437497 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437503 2024-11-24T02:47:17,509 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,509 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,509 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,509 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1220, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,509 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,509 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437503) roll requested 2024-11-24T02:47:17,509 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437509 2024-11-24T02:47:17,514 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,515 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,515 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,515 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,515 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,515 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437503 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437509 2024-11-24T02:47:17,516 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,516 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,516 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,516 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1221, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,516 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,516 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437509) roll requested 2024-11-24T02:47:17,516 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437516 2024-11-24T02:47:17,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,522 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437509 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437516 2024-11-24T02:47:17,523 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,523 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,523 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,523 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1222, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,523 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,523 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437516) roll requested 2024-11-24T02:47:17,523 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437523 2024-11-24T02:47:17,530 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,530 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,530 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,530 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,530 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,530 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437516 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437523 2024-11-24T02:47:17,531 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,531 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,532 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,532 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1223, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,532 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,532 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437523) roll requested 2024-11-24T02:47:17,532 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437532 2024-11-24T02:47:17,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,540 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437523 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437532 2024-11-24T02:47:17,540 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,541 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,541 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,541 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1224, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,541 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,541 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437532) roll requested 2024-11-24T02:47:17,541 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437541 2024-11-24T02:47:17,548 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,548 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,548 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,548 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,548 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,549 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437532 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437541 2024-11-24T02:47:17,549 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,549 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,550 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,550 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1225, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,550 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,550 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437541) roll requested 2024-11-24T02:47:17,550 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437550 2024-11-24T02:47:17,557 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,557 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,557 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,557 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,557 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,557 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437541 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437550 2024-11-24T02:47:17,558 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,558 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,558 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,559 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1226, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,559 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,559 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437550) roll requested 2024-11-24T02:47:17,559 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437559 2024-11-24T02:47:17,566 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,567 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,567 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,567 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,567 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,567 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437550 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437559 2024-11-24T02:47:17,568 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,568 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,568 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,569 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1227, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,569 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,569 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437559) roll requested 2024-11-24T02:47:17,569 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437569 2024-11-24T02:47:17,577 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,577 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,577 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,577 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,577 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,577 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437559 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437569 2024-11-24T02:47:17,578 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,579 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,579 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,579 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1228, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,579 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,579 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437569) roll requested 2024-11-24T02:47:17,579 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437579 2024-11-24T02:47:17,586 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,586 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,586 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,586 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,587 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,587 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437569 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437579 2024-11-24T02:47:17,591 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,591 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,591 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,592 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1229, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,592 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,592 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437579) roll requested 2024-11-24T02:47:17,592 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437592 2024-11-24T02:47:17,599 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,599 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,599 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,599 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,600 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,600 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437579 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437592 2024-11-24T02:47:17,608 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,608 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,609 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,609 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1230, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,609 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,609 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437592) roll requested 2024-11-24T02:47:17,609 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437609 2024-11-24T02:47:17,614 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,614 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,614 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,615 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,615 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,615 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437592 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437609 2024-11-24T02:47:17,620 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,620 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,620 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,621 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1231, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,621 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,621 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437609) roll requested 2024-11-24T02:47:17,621 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437621 2024-11-24T02:47:17,626 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,626 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,626 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,626 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,626 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437609 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437621 2024-11-24T02:47:17,627 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,628 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,628 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,628 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1232, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,628 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,628 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437621) roll requested 2024-11-24T02:47:17,628 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437628 2024-11-24T02:47:17,634 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,634 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,634 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,634 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,634 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,634 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437621 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437628 2024-11-24T02:47:17,637 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,637 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,637 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,637 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1233, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,637 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,637 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437628) roll requested 2024-11-24T02:47:17,637 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437637 2024-11-24T02:47:17,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,642 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,642 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,642 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,642 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437628 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437637 2024-11-24T02:47:17,642 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,642 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,643 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,643 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1234, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,643 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,643 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437637) roll requested 2024-11-24T02:47:17,643 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437643 2024-11-24T02:47:17,647 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,647 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,647 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,647 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,647 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,647 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437637 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437643 2024-11-24T02:47:17,652 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,652 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,652 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,652 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1235, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,652 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,652 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437643) roll requested 2024-11-24T02:47:17,652 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437652 2024-11-24T02:47:17,657 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,657 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,657 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,657 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,657 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,657 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437643 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437652 2024-11-24T02:47:17,658 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,658 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,658 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,658 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1236, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,658 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,658 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437652) roll requested 2024-11-24T02:47:17,658 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437658 2024-11-24T02:47:17,663 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,663 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,663 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,663 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,663 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,663 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437652 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437658 2024-11-24T02:47:17,664 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,664 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,664 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,664 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1237, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,664 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,664 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437658) roll requested 2024-11-24T02:47:17,664 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437664 2024-11-24T02:47:17,669 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,669 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,669 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,669 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,669 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,669 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437658 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437664 2024-11-24T02:47:17,670 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,670 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,670 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,670 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1238, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,670 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,670 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437664) roll requested 2024-11-24T02:47:17,670 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437670 2024-11-24T02:47:17,674 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,674 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,674 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,675 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,675 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,675 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437664 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437670 2024-11-24T02:47:17,675 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,675 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,675 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,675 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1239, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,676 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,676 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437670) roll requested 2024-11-24T02:47:17,676 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437676 2024-11-24T02:47:17,680 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,680 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,680 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,680 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,680 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,680 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437670 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437676 2024-11-24T02:47:17,681 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,681 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,681 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,681 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1240, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,681 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,681 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437676) roll requested 2024-11-24T02:47:17,682 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437681 2024-11-24T02:47:17,686 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,686 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,686 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,686 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,686 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,686 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437676 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437681 2024-11-24T02:47:17,687 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,687 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,687 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,687 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1241, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,688 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,688 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437681) roll requested 2024-11-24T02:47:17,688 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437688 2024-11-24T02:47:17,692 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,692 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,692 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,692 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,692 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,692 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437681 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437688 2024-11-24T02:47:17,692 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,692 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,693 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,693 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1242, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,693 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,693 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437688) roll requested 2024-11-24T02:47:17,693 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437693 2024-11-24T02:47:17,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,697 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,697 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,697 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,698 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437688 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437693 2024-11-24T02:47:17,698 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,698 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,699 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,699 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1243, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,699 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,699 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437693) roll requested 2024-11-24T02:47:17,699 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437699 2024-11-24T02:47:17,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,704 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,704 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,704 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,704 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437693 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437699 2024-11-24T02:47:17,704 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,705 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,705 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,705 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1244, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,705 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,705 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437699) roll requested 2024-11-24T02:47:17,705 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437705 2024-11-24T02:47:17,710 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,710 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,710 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,710 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,710 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,711 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437699 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437705 2024-11-24T02:47:17,711 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,711 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,711 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,712 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1245, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,712 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,712 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437705) roll requested 2024-11-24T02:47:17,712 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437712 2024-11-24T02:47:17,717 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,717 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,717 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,717 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,717 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,717 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437705 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437712 2024-11-24T02:47:17,717 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,718 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,718 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,718 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1246, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,718 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,718 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437712) roll requested 2024-11-24T02:47:17,718 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437718 2024-11-24T02:47:17,722 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,722 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,723 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,723 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,723 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,723 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437712 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437718 2024-11-24T02:47:17,723 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,724 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,724 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,724 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1247, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,724 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,724 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437718) roll requested 2024-11-24T02:47:17,724 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437724 2024-11-24T02:47:17,728 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,728 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,729 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,729 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,729 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,729 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437718 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437724 2024-11-24T02:47:17,729 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,730 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,730 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,730 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1248, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,730 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,730 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437724) roll requested 2024-11-24T02:47:17,730 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437730 2024-11-24T02:47:17,735 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,735 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437724 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437730 2024-11-24T02:47:17,736 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,736 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,736 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,736 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1249, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,736 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,736 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437730) roll requested 2024-11-24T02:47:17,736 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437736 2024-11-24T02:47:17,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,741 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437730 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437736 2024-11-24T02:47:17,744 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,744 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,744 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,744 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1250, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,745 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,745 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437736) roll requested 2024-11-24T02:47:17,745 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437745 2024-11-24T02:47:17,749 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,749 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,749 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,750 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,750 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,750 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437736 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437745 2024-11-24T02:47:17,750 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,750 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,750 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,751 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1251, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,751 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,751 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437745) roll requested 2024-11-24T02:47:17,751 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437751 2024-11-24T02:47:17,755 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,755 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,755 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,755 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,755 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,756 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437745 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437751 2024-11-24T02:47:17,757 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,757 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,758 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,758 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1252, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,758 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,758 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437751) roll requested 2024-11-24T02:47:17,758 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437758 2024-11-24T02:47:17,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,763 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437751 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437758 2024-11-24T02:47:17,763 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,763 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,763 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,764 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1253, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,764 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,764 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437758) roll requested 2024-11-24T02:47:17,764 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437764 2024-11-24T02:47:17,768 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,768 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,768 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,768 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,768 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,768 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437758 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437764 2024-11-24T02:47:17,769 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,769 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,769 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,769 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1254, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,769 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,769 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437764) roll requested 2024-11-24T02:47:17,769 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437769 2024-11-24T02:47:17,774 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,774 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,774 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,774 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,774 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,774 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437764 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437769 2024-11-24T02:47:17,775 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,775 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,775 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,775 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1255, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,775 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,775 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437769) roll requested 2024-11-24T02:47:17,775 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437775 2024-11-24T02:47:17,780 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,780 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,780 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,780 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,780 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,780 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437769 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437775 2024-11-24T02:47:17,780 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,780 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,781 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,781 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1256, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,781 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,781 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437775) roll requested 2024-11-24T02:47:17,781 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437781 2024-11-24T02:47:17,785 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,785 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,785 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,785 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,785 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,785 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437775 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437781 2024-11-24T02:47:17,786 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,786 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,786 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,786 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1257, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,786 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,786 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437781) roll requested 2024-11-24T02:47:17,786 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437786 2024-11-24T02:47:17,792 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,792 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,792 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,792 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,792 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,792 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437781 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437786 2024-11-24T02:47:17,793 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,793 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,793 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,793 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1258, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,793 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,793 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437786) roll requested 2024-11-24T02:47:17,794 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437793 2024-11-24T02:47:17,801 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,801 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,802 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,802 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,802 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,802 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437786 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437793 2024-11-24T02:47:17,804 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,804 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,804 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,805 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1259, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,805 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,805 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437793) roll requested 2024-11-24T02:47:17,805 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437805 2024-11-24T02:47:17,813 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,813 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,813 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,813 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,813 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,813 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437793 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437805 2024-11-24T02:47:17,814 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,814 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,814 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,814 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1260, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,814 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,814 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437805) roll requested 2024-11-24T02:47:17,815 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437815 2024-11-24T02:47:17,823 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,823 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,823 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,823 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,823 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,823 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437805 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437815 2024-11-24T02:47:17,824 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,824 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,824 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,825 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1261, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,825 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,825 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437815) roll requested 2024-11-24T02:47:17,825 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437825 2024-11-24T02:47:17,830 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,830 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,830 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,830 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,830 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,830 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437815 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437825 2024-11-24T02:47:17,831 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,831 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,831 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,831 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1262, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,831 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,831 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437825) roll requested 2024-11-24T02:47:17,831 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437831 2024-11-24T02:47:17,836 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,836 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,836 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,836 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,836 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,836 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437825 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437831 2024-11-24T02:47:17,837 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,837 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,837 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,837 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1263, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,837 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,837 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437831) roll requested 2024-11-24T02:47:17,837 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437837 2024-11-24T02:47:17,842 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,842 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,842 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,842 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,842 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,842 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437831 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437837 2024-11-24T02:47:17,844 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,844 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,844 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,844 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1264, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,845 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,845 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437837) roll requested 2024-11-24T02:47:17,845 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437845 2024-11-24T02:47:17,849 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,850 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,850 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,850 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,850 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,850 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437837 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437845 2024-11-24T02:47:17,852 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,852 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,852 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,852 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1265, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,852 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,852 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437845) roll requested 2024-11-24T02:47:17,852 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437852 2024-11-24T02:47:17,857 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,857 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,857 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,857 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,857 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,857 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437845 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437852 2024-11-24T02:47:17,858 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,858 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,858 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,858 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1266, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,858 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,858 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437852) roll requested 2024-11-24T02:47:17,858 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437858 2024-11-24T02:47:17,863 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,863 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,863 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,863 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,863 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,863 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437852 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437858 2024-11-24T02:47:17,864 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,864 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,864 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,865 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1267, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,865 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,865 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437858) roll requested 2024-11-24T02:47:17,865 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437865 2024-11-24T02:47:17,869 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,869 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,869 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,869 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,869 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,869 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437858 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437865 2024-11-24T02:47:17,870 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,870 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,870 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,870 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1268, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,870 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,870 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437865) roll requested 2024-11-24T02:47:17,870 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437870 2024-11-24T02:47:17,874 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,874 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,874 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,875 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,875 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,875 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437865 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437870 2024-11-24T02:47:17,875 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,875 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,875 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,876 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1269, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,876 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,876 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437870) roll requested 2024-11-24T02:47:17,876 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437876 2024-11-24T02:47:17,880 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,880 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,880 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,880 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,880 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,880 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437870 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437876 2024-11-24T02:47:17,884 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,884 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,884 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,884 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1270, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,884 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,884 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437876) roll requested 2024-11-24T02:47:17,885 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437885 2024-11-24T02:47:17,890 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,890 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,890 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,890 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,890 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,891 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437876 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437885 2024-11-24T02:47:17,892 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,892 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,892 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,892 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1271, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,892 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,892 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437885) roll requested 2024-11-24T02:47:17,892 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437892 2024-11-24T02:47:17,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,899 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437885 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437892 2024-11-24T02:47:17,900 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,901 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1272, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,901 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,901 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,901 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,901 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437892) roll requested 2024-11-24T02:47:17,901 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437901 2024-11-24T02:47:17,907 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,907 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,907 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,907 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,907 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,907 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437892 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437901 2024-11-24T02:47:17,908 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,908 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,908 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,909 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1273, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,909 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,909 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437901) roll requested 2024-11-24T02:47:17,909 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437909 2024-11-24T02:47:17,915 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,916 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,916 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,916 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,916 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,916 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437901 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437909 2024-11-24T02:47:17,917 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,917 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,917 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,917 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1274, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,917 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,917 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437909) roll requested 2024-11-24T02:47:17,918 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437918 2024-11-24T02:47:17,925 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,925 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,925 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,925 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,925 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,925 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437909 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437918 2024-11-24T02:47:17,927 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,927 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,927 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,927 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1275, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,927 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,927 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437918) roll requested 2024-11-24T02:47:17,928 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437928 2024-11-24T02:47:17,935 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,935 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,935 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,935 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,935 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,936 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437918 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437928 2024-11-24T02:47:17,937 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,937 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,937 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,938 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1276, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,938 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,938 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437928) roll requested 2024-11-24T02:47:17,938 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437938 2024-11-24T02:47:17,946 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,946 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,946 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,946 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,946 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,946 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437928 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437938 2024-11-24T02:47:17,948 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,948 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,948 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,948 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1277, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,948 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,948 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437938) roll requested 2024-11-24T02:47:17,949 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437949 2024-11-24T02:47:17,956 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,956 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,956 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,956 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,956 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,956 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437938 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437949 2024-11-24T02:47:17,957 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,957 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,957 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,957 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1278, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,957 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,957 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437949) roll requested 2024-11-24T02:47:17,957 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437957 2024-11-24T02:47:17,962 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,962 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,962 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,962 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,962 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,962 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437949 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437957 2024-11-24T02:47:17,963 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,963 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,963 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,964 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1279, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,964 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,964 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437957) roll requested 2024-11-24T02:47:17,964 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437964 2024-11-24T02:47:17,970 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,970 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,970 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,970 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,970 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,970 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437957 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437964 2024-11-24T02:47:17,971 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,971 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,971 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,971 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1280, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,971 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,971 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437964) roll requested 2024-11-24T02:47:17,971 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437971 2024-11-24T02:47:17,977 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,977 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,977 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,977 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,977 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,977 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437964 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437971 2024-11-24T02:47:17,978 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,978 ERROR [sync.1 {}] wal.FSHLog$SyncRunner(463): Error syncing java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,978 WARN [FSHLog-0-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync-prefix:default {}] wal.AbstractFSWAL(1504): sync failed java.io.IOException: FAKE! Failed to replace a bad datanode... at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog$1.sync(TestFailedAppendAndSync.java:158) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:461) ~[classes/:?] 2024-11-24T02:47:17,979 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1281, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,979 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,979 DEBUG [LogRoller {}] wal.AbstractWALRoller(198): WAL DodgyFSLog wal:(num 1732416437971) roll requested 2024-11-24T02:47:17,979 INFO [LogRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732416437979 2024-11-24T02:47:17,979 ERROR [Time-limited test {}] wal.AbstractFSWAL(939): Failed log archiving for the log /a/b, java.io.IOException: throw archival exception at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archiveLogFile(TestFailedAppendAndSync.java:137) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.archive(AbstractFSWAL.java:934) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.archive(TestFailedAppendAndSync.java:143) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync.testLockupAroundBadAssignSync(TestFailedAppendAndSync.java:280) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T02:47:17,982 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e6e36326512c58b59a3d26a05c314372, disabling compactions & flushes 2024-11-24T02:47:17,982 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testLockupAroundBadAssignSync,,1732416407245.e6e36326512c58b59a3d26a05c314372. 2024-11-24T02:47:17,982 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testLockupAroundBadAssignSync,,1732416407245.e6e36326512c58b59a3d26a05c314372. 2024-11-24T02:47:17,983 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testLockupAroundBadAssignSync,,1732416407245.e6e36326512c58b59a3d26a05c314372. after waiting 0 ms 2024-11-24T02:47:17,983 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testLockupAroundBadAssignSync,,1732416407245.e6e36326512c58b59a3d26a05c314372. 2024-11-24T02:47:17,983 WARN [LogRoller {}] util.Shell(1106): Interrupted while joining on: Thread[Thread-5792,5,] java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Thread.join(Thread.java:1313) ~[?:?] at java.lang.Thread.join(Thread.java:1381) ~[?:?] at org.apache.hadoop.util.Shell.joinThread(Shell.java:1103) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1063) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1377) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.execCommand(Shell.java:1359) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.setPermission(RawLocalFileSystem.java:1116) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:454) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem$LocalFSFileOutputStream.(RawLocalFileSystem.java:414) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.createOutputStreamWithMode(RawLocalFileSystem.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:566) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.RawLocalFileSystem.create(RawLocalFileSystem.java:597) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem$ChecksumFSOutputSummer.(ChecksumFileSystem.java:633) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.create(ChecksumFileSystem.java:721) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.ChecksumFileSystem.createNonRecursive(ChecksumFileSystem.java:765) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystem$FileSystemDataOutputStreamBuilder.build(FileSystem.java:4705) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.wal.ProtobufLogWriter.initOutput(ProtobufLogWriter.java:117) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractProtobufLogWriter.init(AbstractProtobufLogWriter.java:171) ~[classes/:?] at org.apache.hadoop.hbase.wal.FSHLogProvider.createWriter(FSHLogProvider.java:82) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.FSHLog.createWriterInstance(FSHLog.java:259) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:148) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.createWriterInstance(TestFailedAppendAndSync.java:114) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriterInternal(AbstractFSWAL.java:1099) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$rollWriter$9(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.rollWriter(AbstractFSWAL.java:1136) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.TestFailedAppendAndSync$DodgyFSLog.rollWriter(TestFailedAppendAndSync.java:129) ~[test-classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller$RollController.rollWal(AbstractWALRoller.java:311) ~[classes/:?] at org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:212) ~[classes/:?] 2024-11-24T02:47:17,985 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 83 in region testLockupAroundBadAssignSync,,1732416407245.e6e36326512c58b59a3d26a05c314372. 2024-11-24T02:47:17,985 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testLockupAroundBadAssignSync,,1732416407245.e6e36326512c58b59a3d26a05c314372. 2024-11-24T02:47:17,985 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e6e36326512c58b59a3d26a05c314372: Waiting for close lock at 1732416437982Disabling compacts and flushes for region at 1732416437982Disabling writes for close at 1732416437983 (+1 ms)Writing region close event to WAL at 1732416437984 (+1 ms)Closed at 1732416437985 (+1 ms) 2024-11-24T02:47:17,988 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,988 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,988 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,989 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,989 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,989 INFO [LogRoller {}] wal.AbstractFSWAL(987): Rolled WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437971 with entries=3, filesize=577 B; new WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea/TestHRegiontestLockupAroundBadAssignSync/testLockupAroundBadAssignSync/wal.1732416437979 2024-11-24T02:47:17,990 DEBUG [LogRoller {}] wal.AbstractFSWAL(1109): Create new DodgyFSLog writer with pipeline: [] 2024-11-24T02:47:17,990 INFO [LogRoller {}] wal.AbstractFSWAL(843): Too many WALs; count=1282, max=55; forcing (partial) flush of 1 region(s): e6e36326512c58b59a3d26a05c314372[MyCF] 2024-11-24T02:47:17,990 WARN [LogRoller {}] regionserver.LogRoller(45): Failed to schedule flush of e6e36326512c58b59a3d26a05c314372, because it is not online on us 2024-11-24T02:47:17,990 INFO [LogRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T02:47:17,990 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,990 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,990 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,990 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:17,990 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T02:47:18,218 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: DodgyFSLog wal:(num 1732416437979) 2024-11-24T02:47:18,219 INFO [Time-limited test {}] regionserver.TestFailedAppendAndSync(105): Cleaning test directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/14f19692-1130-5146-d3a3-7df0e07fe1ea 2024-11-24T02:47:18,693 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.TestFailedAppendAndSync#testLockupAroundBadAssignSync Thread=22 (was 11) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=287 (was 287), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=696 (was 888), ProcessCount=11 (was 11), AvailableMemoryMB=10467 (was 11238)